diff --git a/.github/workflows/host-tests.yml b/.github/workflows/host-tests.yml index f16fb33953f7c..fd665f948d991 100644 --- a/.github/workflows/host-tests.yml +++ b/.github/workflows/host-tests.yml @@ -57,6 +57,8 @@ jobs: command -v "$p" >/dev/null || need="$need $p" done command -v "ld.lld-$CLANG_VERSION" >/dev/null || need="$need lld-$CLANG_VERSION" + # Device-tree tests read the built trees with fdtget. + command -v fdtget >/dev/null || need="$need device-tree-compiler" dpkg -s "libclang-rt-$CLANG_VERSION-dev" >/dev/null 2>&1 || need="$need libclang-rt-$CLANG_VERSION-dev" if [ -n "$need" ]; then sudo apt-get update -q diff --git a/Documentation/ABI/testing/debugfs-driver-asahi-stats b/Documentation/ABI/testing/debugfs-driver-asahi-stats new file mode 100644 index 0000000000000..f795d25ace2ba --- /dev/null +++ b/Documentation/ABI/testing/debugfs-driver-asahi-stats @@ -0,0 +1,47 @@ +What: /sys/kernel/debug/dri//asahi_stats +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only, root-only dump of the raw AGX firmware statistics + that the drm/asahi driver receives on its statistics channel. + The driver stores the latest message of each kind and prints + it unchanged. It does not compute utilization or any other + derived value. + + The file is created in the debugfs directory of the DRM + device when the device is registered. It also appears under + the other names of that directory (dri/128 and the platform + device name, for example dri/406400000.gpu). + + The format is ASCII text, one line per record: + + - line 1: asahi_stats_v1, the format tag. A reader must check it. + - line 2: device= + - line 3: now_ns= + - line 4: messages count= + - then one line per message kind, in this order: power, + power_on, power_off, utilization, avg_power, temperature, + power_state, fw_busy, pstate, temp_sensor. + + A kind line is " count=" followed, when n is not 0, + by " captured_ns=" and the fields of + that kind as key=value pairs: + + - power: power + - power_on: off_time + - power_off: on_time + - utilization: timestamp, util1, util2, util3, util4 + - avg_power: active_cs, unk2, unk3, unk4, avg_power + - temperature: raw_value, scale, tmin, tmax + - power_state: timestamp, last_busy_ts, active, poweroff, + unk1, pstate, unk2, unk3 + - fw_busy: timestamp, busy + - pstate: ps_min, unk1, ps_max, unk2 + - temp_sensor: sensor_id, raw_value, scale, tmin, tmax + + All counts are cumulative since the driver bound the device. A rate needs two reads. + The values are the raw firmware values. Their units and the + meaning of the util*, unk* fields are not established. A kind + that was never received prints count=0 and no payload. Keys + may be added in later versions of the format tag; readers + must ignore keys they do not know. diff --git a/arch/arm64/boot/dts/apple/t6030.dtsi b/arch/arm64/boot/dts/apple/t6030.dtsi index 685b67606dc87..c19d3691f9227 100644 --- a/arch/arm64/boot/dts/apple/t6030.dtsi +++ b/arch/arm64/boot/dts/apple/t6030.dtsi @@ -836,6 +836,8 @@ #clock-cells = <1>; }; + /* The display handoff gate installs PMP unless explicitly static. */ + #ifdef APPLE_USE_PMP pmp_dart: iommu@350300000 { compatible = "apple,t6030-dart", "apple,t8110-dart"; reg = <0x3 0x50300000 0x0 0x4000>; @@ -985,6 +987,8 @@ power-domains = <&ps_pmp>, <&ps_pms_sram>; }; + #endif /* APPLE_USE_PMP */ + aic: interrupt-controller@351000000 { compatible = "apple,t6030-aic3", "apple,t8122-aic3"; #interrupt-cells = <3>; diff --git a/arch/arm64/boot/dts/apple/t6031-dcp.dtsi b/arch/arm64/boot/dts/apple/t6031-dcp.dtsi index 6d3b53cd95e57..f679c57b97709 100644 --- a/arch/arm64/boot/dts/apple/t6031-dcp.dtsi +++ b/arch/arm64/boot/dts/apple/t6031-dcp.dtsi @@ -9,9 +9,10 @@ * interrupts and DART windows are those of the J516C ADT. * * Every node stays status = "disabled". The display gate enables the DCP, - * its mailbox, the two DARTs and the display subsystem only when - * apple_t6031_display.enable=1 is on the kernel command line and the boot - * loader has set apple,t6031-handoff = <1>. Compatibles are t6031-specific, + * its mailbox, the two DARTs and the display subsystem only when the boot + * loader has set apple,t6031-handoff = <1> and the native display is + * requested: by default on the J516C, with apple_t6031_display.enable=1 on + * the other boards, never with apple_t6031_display.enable=0. Compatibles are t6031-specific, * so a kernel that does not know them leaves the nodes unbound. * * Only the T6031 board files include this file: t6034.dtsi includes diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d71c5c0370f2e..007ce2913c158 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -64,28 +64,44 @@ struct apple_drm_private { }; /* Component readiness is later than platform probe/bind. */ -static DEFINE_MUTEX(apple_t6030_display_lock); +static DEFINE_MUTEX(apple_g15_display_lock); static struct device *apple_t6030_display; +static struct device *apple_t6031_display; -static int apple_t6030_display_publish(struct device *dev) +/* The publication slot of a G15 SoC's internal display subsystem, or NULL. */ +static struct device **apple_g15_display_slot(struct device *dev) { - if (!of_device_is_compatible(dev->of_node, "apple,t6030-display-subsystem")) + if (of_device_is_compatible(dev->of_node, "apple,t6030-display-subsystem")) + return &apple_t6030_display; + if (of_device_is_compatible(dev->of_node, "apple,t6031-display-subsystem")) + return &apple_t6031_display; + return NULL; +} + +static int apple_g15_display_publish(struct device *dev) +{ + struct device **slot = apple_g15_display_slot(dev); + + if (!slot) return 0; - guard(mutex)(&apple_t6030_display_lock); - if (apple_t6030_display) - return apple_t6030_display == dev ? 0 : -EBUSY; - apple_t6030_display = get_device(dev); + guard(mutex)(&apple_g15_display_lock); + if (*slot) + return *slot == dev ? 0 : -EBUSY; + *slot = get_device(dev); return 0; } -static void apple_t6030_display_withdraw(struct device *dev) +static void apple_g15_display_withdraw(struct device *dev) { + struct device **slot = apple_g15_display_slot(dev); struct device *old = NULL; - scoped_guard(mutex, &apple_t6030_display_lock) { - if (apple_t6030_display == dev) { - old = apple_t6030_display; - apple_t6030_display = NULL; + if (!slot) + return; + scoped_guard(mutex, &apple_g15_display_lock) { + if (*slot == dev) { + old = *slot; + *slot = NULL; } } put_device(old); @@ -96,22 +112,35 @@ static void apple_t6030_display_withdraw(struct device *dev) * publish separately. This lock protects only publication and a device link; * no display calls or firmware RPC run while it is held. */ -int apple_drm_t6030_link_device(struct device *consumer) +static int apple_g15_link_device(struct device *consumer, const char *machine, + const char *gpu, struct device **display) { struct device_link *link; - if (!of_machine_is_compatible("apple,t6030") || !consumer->of_node || - !of_device_is_compatible(consumer->of_node, "apple,agx-t6030")) + if (!of_machine_is_compatible(machine) || !consumer->of_node || + !of_device_is_compatible(consumer->of_node, gpu)) return -ENODEV; - guard(mutex)(&apple_t6030_display_lock); - if (!apple_t6030_display) + guard(mutex)(&apple_g15_display_lock); + if (!*display) return -EPROBE_DEFER; - link = device_link_add(consumer, apple_t6030_display, - DL_FLAG_AUTOREMOVE_CONSUMER); + link = device_link_add(consumer, *display, DL_FLAG_AUTOREMOVE_CONSUMER); return link ? 0 : -EINVAL; } + +int apple_drm_t6030_link_device(struct device *consumer) +{ + return apple_g15_link_device(consumer, "apple,t6030", "apple,agx-t6030", + &apple_t6030_display); +} EXPORT_SYMBOL_GPL(apple_drm_t6030_link_device); +int apple_drm_t6031_link_device(struct device *consumer) +{ + return apple_g15_link_device(consumer, "apple,t6031", "apple,agx-t6031", + &apple_t6031_display); +} +EXPORT_SYMBOL_GPL(apple_drm_t6031_link_device); + DEFINE_DRM_GEM_DMA_FOPS(apple_fops); #define DART_PAGE_SIZE 16384 @@ -1316,7 +1345,7 @@ static int apple_drm_init(struct device *dev) drm_client_setup_with_fourcc(&apple->drm, DRM_FORMAT_XRGB8888); /* A subsequent successful platform probe retries deferred GPU probes. */ - ret = apple_t6030_display_publish(dev); + ret = apple_g15_display_publish(dev); if (ret) { drm_dev_unregister(&apple->drm); goto err_unbind; @@ -1338,7 +1367,7 @@ static void apple_drm_uninit(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - apple_t6030_display_withdraw(dev); + apple_g15_display_withdraw(dev); /* An aborted takeover leaves its DCPs running until the reboot. */ if (apple->aborted) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 42843e3459217..decf16ea09647 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -36,6 +36,7 @@ mod queue; mod regs; mod slotalloc; mod stats; +mod stats_raw; mod sysfs_exports; mod timestamp; mod usage; @@ -170,7 +171,7 @@ module! { }, g15_pb_max_blocks: u32 { default: 3072, - description: "G15: parameter-buffer growth maximum given to the firmware, blocks of 128 KiB (1024..4095 in effect; the blocks above it, up to 4095, answer partial renders; the 512 MiB heap is allocated either way)", + description: "G15: PB growth maximum given to the firmware, 128 KiB blocks (64..4095)", }, g15_early_tiling: u32 { default: 0, diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 249b10bb361ca..5a14982dca053 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -24,13 +24,17 @@ use crate::{ event, gpu, mem, - module_parameters, // + module_parameters, + stats_raw::RawStatsSnapshot, // }; use core::sync::atomic::Ordering; use kernel::{ c_str, prelude::*, - sync::Arc, + sync::{ + Arc, + SpinLock, // + }, time::{ delay::fsleep, Delta, @@ -588,6 +592,9 @@ pub(crate) struct StatsChannel { dev: AsahiDevRef, ch: RxChannel, snap: Arc, + /// Last raw sample of every statistics kind, shared with the export + /// through `crate::stats_raw`. + raw: Arc>, /// Rate of firmware stats-message timestamps (Hz). All stats timestamps /// run on the chip base clock (`HwConfig::base_clock_hz`, 24 MHz on every /// supported SoC), not nanoseconds. @@ -604,12 +611,14 @@ impl StatsChannel::ver { dev: AsahiDevice, alloc: &mut gpu::KernelAllocators, snap: Arc, + raw: Arc>, ts_hz: u64, ) -> Result { Ok(StatsChannel::ver { dev: dev.clone(), ch: RxChannel::::new(alloc, 0x100)?, snap, + raw, ts_hz, last_util_ts: 0, }) @@ -631,6 +640,7 @@ impl StatsChannel::ver { // accessing the enum view is valid. let msg = unsafe { msg.msg }; cls_dev_dbg!(StatsCh, self.dev, "Stats: {:?}\n", msg); + self.record_raw(&msg); if module_parameters::stats_export.value() != 0 { // Retain the latest values for /sys/class/drm/card*/ // device/agx_stats. Read-only with respect to @@ -713,4 +723,80 @@ impl StatsChannel::ver { } } } + + /// Record one statistics message in the raw per-device snapshot. + /// + /// The firmware words are stored exactly as they arrived: no unit is + /// assumed, nothing is scaled and no metric is derived from them. + fn record_raw(&mut self, msg: &StatsMsg::ver) { + let stamp = crate::stats_raw::monotonic_ns(); + let mut snap = self.raw.lock(); + snap.note_message(); + match msg { + StatsMsg::ver::Power { power, .. } => snap.set_power(power.0, stamp), + StatsMsg::ver::PowerOn { off_time, .. } => snap.set_power_on(off_time.0, stamp), + StatsMsg::ver::PowerOff { on_time, .. } => snap.set_power_off(on_time.0, stamp), + StatsMsg::ver::Utilization { + timestamp, + util1, + util2, + util3, + util4, + } => snap.set_utilization(timestamp.0, [*util1, *util2, *util3, *util4], stamp), + StatsMsg::ver::AvgPower { + active_cs, + unk2, + unk3, + unk4, + avg_power, + } => snap.set_avg_power(active_cs.0, [*unk2, *unk3, *unk4], *avg_power, stamp), + StatsMsg::ver::Temperature { + raw_value, + scale, + tmin, + tmax, + .. + } => snap.set_temperature(*raw_value, *scale, *tmin, *tmax, stamp), + StatsMsg::ver::PowerState { + timestamp, + last_busy_ts, + active, + poweroff, + unk1, + pstate, + unk2, + unk3, + } => snap.set_power_state( + timestamp.0, + last_busy_ts.0, + *active, + *poweroff, + *unk1, + *pstate, + *unk2, + *unk3, + stamp, + ), + StatsMsg::ver::FwBusy { timestamp, busy } => { + snap.set_fw_busy(timestamp.0, *busy, stamp) + } + StatsMsg::ver::PState { + ps_min, + unk1, + ps_max, + unk2, + .. + } => snap.set_pstate(*ps_min, *unk1, *ps_max, *unk2, stamp), + StatsMsg::ver::TempSensor { + sensor_id, + raw_value, + scale, + tmin, + tmax, + .. + } => snap.set_temp_sensor(*sensor_id, *raw_value, *scale, *tmin, *tmax, stamp), + // The remaining tags are not decoded by the driver. + _ => {} + } + } } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index a89728cc2f2b3..d6e461a7c3210 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -302,12 +302,20 @@ impl platform::Driver for AsahiPlatformDriver { gpu.init()?; let stats = gpu.stats_snapshot(); + let debug_stats = gpu.raw_stats_snapshot(); let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is // unbound; it is never forgotten. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + // The DRM core created this card's debugfs directory when the device was registered; + // publish the raw firmware statistics in it. drm_dev_unregister() removes the file, and + // waits for its readers, before the registration data that owns the snapshot is dropped. + if let Some(snapshot) = debug_stats.as_ref() { + crate::stats_raw::register(reg.device(), snapshot); + } + // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); diff --git a/drivers/gpu/drm/asahi/g15/adt.rs b/drivers/gpu/drm/asahi/g15/adt.rs index aece4f083c0ad..83807d985866a 100644 --- a/drivers/gpu/drm/asahi/g15/adt.rs +++ b/drivers/gpu/drm/asahi/g15/adt.rs @@ -30,10 +30,13 @@ //! one cell per cluster). The same admission applies, without the DRAM bounds, and the four UAT //! regions may lie in System RAM: they are reserved-memory nodes without `no-map`, which the //! kernel maps, so the driver reaches them through its own mapping ([`super::regs::Mapping`]). +//! The GPU's sensor rows, which the node does not carry, come from the same ADT export as on +//! T6030: `/arm-io/aon-ptd` `ptd_data_table` ([`super::board::record::sensor_rows`]). use core::ops::Range; use kernel::{c_str, device, prelude::*}; +use super::board::record; use super::boot::glue; use super::hw; @@ -57,12 +60,13 @@ pub(crate) struct GpuAdt { pub(crate) perf_mhz: [u32; PERF_STATES], pub(crate) perf_mv: [u32; PERF_STATES], pub(crate) sram_mv: [u32; PERF_STATES], - /// Per-cluster core mV of each state, rounded up from the device tree's microvolts, how - /// many voltage cells the node has (1 on the ADT path), and the state's power in mW (0 on - /// the ADT path). + /// Per-cluster core mV of each state, rounded up from the device tree's microvolts, and how + /// many voltage cells the node has (1 on the ADT path). pub(crate) cluster_mv: [[u32; CLUSTERS]; PERF_STATES], pub(crate) cluster_cells: usize, - pub(crate) perf_mw: [u32; PERF_STATES], + /// The GPU's six sensor rows of the ADT's `aon-ptd` table (T6031; zero on the T6030 path, + /// whose templates carry their own). + pub(crate) sensor_rows: [[u8; 12]; record::SENSOR_ROW_COUNT], } /// Voltage cells the layout keeps per state. @@ -328,7 +332,7 @@ fn parse(dev: &device::Device, adt: &Adt<'_>) -> Result { sram_mv: [0; PERF_STATES], cluster_mv: [[0; CLUSTERS]; PERF_STATES], cluster_cells: 1, - perf_mw: [0; PERF_STATES], + sensor_rows: [[0; 12]; record::SENSOR_ROW_COUNT], }; collect_perf(dev, adt, sgx, &mut out)?; for i in 0..PERF_STATES { @@ -437,11 +441,11 @@ fn from_device_tree(dev: &device::Device) -> Result { sram_mv: [0; PERF_STATES], cluster_mv: [[0; CLUSTERS]; PERF_STATES], cluster_cells: 0, - perf_mw: [0; PERF_STATES], + sensor_rows: [[0; 12]; record::SENSOR_ROW_COUNT], }; // Operating points: MHz, the highest cluster voltage of each and the clusters' voltages - // (rounded up) and power for the hardware record; the SRAM voltage is not in the device - // tree. + // (rounded up) for the hardware record; the SRAM voltage is not in the device tree, and + // the powers are a boot-loader stand-in. let opps = node .parse_phandle(c_str!("operating-points-v2"), 0) .ok_or(EINVAL) @@ -450,7 +454,6 @@ fn from_device_tree(dev: &device::Device) -> Result { for opp in opps.children() { let hz: u64 = opp.get_property(c_str!("opp-hz"))?; let uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; - let uw: Option = opp.get_opt_property(c_str!("opp-microwatt"))?; let mv = uv.iter().max().copied().unwrap_or(0) / 1000; if uv.is_empty() || uv.len() > CLUSTERS || (count > 0 && uv.len() != out.cluster_cells) { dev_err!(dev, "GPU operating point {} has {} voltage cells\n", count, uv.len()); @@ -467,7 +470,6 @@ fn from_device_tree(dev: &device::Device) -> Result { out.cluster_mv[count][c] = v.div_ceil(1000); } out.cluster_cells = uv.len(); - out.perf_mw[count] = uw.unwrap_or(0) / 1000; } count += 1; } @@ -480,19 +482,44 @@ fn from_device_tree(dev: &device::Device) -> Result { ); return Err(EINVAL); } + out.sensor_rows = sensor_rows(dev)?; Ok(out) } +/// A private copy of the boot loader's ADT export. +fn copy() -> Result> { + let (start, size) = glue::adt_phram()?; + if size == 0 || size > 0x200000 || start & 0x3fff != 0 { + return Err(EINVAL); + } + super::identity::copy_physical(start, usize::try_from(size)?) +} + +/// The GPU's six sensor rows: `/arm-io/aon-ptd` `ptd_data_table` of the ADT export +/// ([`record::sensor_rows`]), which the device tree does not carry. +fn sensor_rows(dev: &device::Device) -> Result<[[u8; 12]; record::SENSOR_ROW_COUNT]> { + let copy = + copy().inspect_err(|_| dev_err!(dev, "no usable ADT export for the GPU sensor rows\n"))?; + let adt = Adt { data: © }; + let table = adt + .node_end(0, 0) + .and_then(|_| adt.child(0, "arm-io")) + .and_then(|armio| adt.child(armio, "aon-ptd")) + .and_then(|ptd| adt.prop(ptd, "ptd_data_table")) + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "ADT has no aon-ptd ptd_data_table\n"))?; + record::sensor_rows(table).map_err(|why| { + dev_err!(dev, "ADT aon-ptd: {}\n", why); + EINVAL + }) +} + /// Reads and admits the GPU's regions, segments and performance states (read-only), from the /// source the SoC configuration names. pub(crate) fn read(dev: &device::Device, cfg: &super::Config) -> Result { if cfg.platform == hw::Platform::DeviceTree { return from_device_tree(dev); } - let (start, size) = glue::adt_phram()?; - if size == 0 || size > 0x200000 || start & 0x3fff != 0 { - return Err(EINVAL); - } - let copy = super::identity::copy_physical(start, usize::try_from(size)?)?; + let copy = copy()?; parse(dev, &Adt { data: © }) } diff --git a/drivers/gpu/drm/asahi/g15/board.rs b/drivers/gpu/drm/asahi/g15/board.rs index da0a7408575bf..f9682b2ead10c 100644 --- a/drivers/gpu/drm/asahi/g15/board.rs +++ b/drivers/gpu/drm/asahi/g15/board.rs @@ -10,11 +10,11 @@ //! against its hash at boot. //! //! A SoC whose board data comes from its device tree ([`super::hw::Platform::DeviceTree`]) gets -//! its hardware record from [`hardware_record`] ([`record`]) and its Globals and Power from -//! [`device_tree`]: the G15S templates with the GPU node's power words and the hardware -//! record's maximum power ([`words`]). +//! its hardware record from [`hardware_record`] ([`record`]: the GPU node, the ADT's sensor +//! rows and the SoC's reference words) and its Globals and Power from [`device_tree`]: the G15S +//! templates with the GPU node's power words and the hardware record's maximum power +//! ([`words`]). -pub(crate) mod blob; pub(crate) mod data; pub(crate) mod record; pub(crate) mod template; @@ -98,7 +98,6 @@ fn aux_opps( mhz, mv: [uv[0].div_ceil(1000), 0, 0, 0], cells: 1, - mw: 0, }; n += 1; } @@ -110,13 +109,13 @@ fn aux_opps( } /// The hardware record of a device-tree SoC ([`record`]): built from the GPU node's properties -/// and operating-point tables, the layout's IOMapping table and the ADT regions, with the -/// words the device tree does not carry copied from `file`, the machine's own record. +/// and operating-point tables, the layout's IOMapping table, the ADT regions and sensor rows, +/// and the SoC's `reference` words, which neither carries. pub(crate) fn hardware_record( dev: &device::Device, cfg: &super::Config, adt: &GpuAdt, - file: &[u8], + reference: &record::Reference, ) -> Result> { let layout = cfg.layout; let node = dev.of_node().ok_or(ENODEV)?; @@ -136,7 +135,6 @@ pub(crate) fn hardware_record( mhz: adt.perf_mhz[i], mv: adt.cluster_mv[i], cells: adt.cluster_cells, - mw: adt.perf_mw[i], }; } let mut cs = [record::Opp::default(); 16]; @@ -159,12 +157,6 @@ pub(crate) fn hardware_record( writable, }; } - let max_power_mw = u32::from_le_bytes( - file.get(record::MAX_POWER..record::MAX_POWER + 4) - .ok_or(EINVAL)? - .try_into() - .map_err(|_| EINVAL)?, - ); let inputs = record::Inputs { props, zones: &zones, @@ -177,15 +169,14 @@ pub(crate) fn hardware_record( usc_base: hw::client::COMPACT_WINDOW.start, user_end: hw::USER_RANGE.end, timestamps: va::TIMESTAMPS, - max_power_mw, + sensor_rows: adt.sensor_rows, + reference, }; let mut out = KVVec::from_elem(0u8, record::SIZE, GFP_KERNEL)?; - record::from_file(&mut out, file) - .and_then(|()| record::build(&mut out, &inputs)) - .map_err(|why| { - dev_err!(dev, "hardware record from the device tree: {}\n", why); - EINVAL - })?; + record::build(&mut out, &inputs).map_err(|why| { + dev_err!(dev, "hardware record from the device tree: {}\n", why); + EINVAL + })?; Ok(out) } diff --git a/drivers/gpu/drm/asahi/g15/board/blob.rs b/drivers/gpu/drm/asahi/g15/board/blob.rs deleted file mode 100644 index 04cff8bbaa3d9..0000000000000 --- a/drivers/gpu/drm/asahi/g15/board/blob.rs +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Board images from a firmware file. -//! -//! A SoC whose board images are not built-in templates names a firmware file -//! ([`super::super::hw::Board::Blob`]), built into the kernel with `CONFIG_EXTRA_FIRMWARE` or -//! loaded from `/lib/firmware`, and pins its SHA-256. The file holds the three images as the -//! firmware of that machine accepted them; the build then writes the pointers and policy words -//! it owns into them (TTBAT, IOMapping VAs, the timestamp arena, the idle policy). -//! -//! Layout, little-endian: the magic `T6031BRD`, u32 format version (1), u32 image count (3), -//! then per image `u32 kind, u32 size` with kinds 0 (HardwareData), 1 (Globals) and -//! 2 (Power), then the images' bytes in that order. Sizes must match the SoC's layout. - -use kernel::{device, firmware::Firmware, prelude::*}; - -use super::super::boot::glue; -use super::super::identity::Hex; - -const MAGIC: &[u8; 8] = b"T6031BRD"; -const VERSION: u32 = 1; -const IMAGES: usize = 3; - -/// The three images. -pub(crate) struct Images { - pub(crate) hardware: KVVec, - pub(crate) globals: KVVec, - pub(crate) power: KVVec, -} - -fn le32(b: &[u8], off: usize) -> Result { - Ok(u32::from_le_bytes( - b.get(off..off + 4) - .ok_or(EINVAL)? - .try_into() - .map_err(|_| EINVAL)?, - )) -} - -fn copy(b: &[u8], range: core::ops::Range) -> Result> { - let src = b.get(range).ok_or(EINVAL)?; - let mut out = KVVec::from_elem(0u8, src.len(), GFP_KERNEL)?; - out.copy_from_slice(src); - Ok(out) -} - -/// Requests `name`, verifies its digest against `sha256` and splits it into the images, each -/// of the size `sizes` gives for its kind. -pub(crate) fn load( - dev: &device::Device, - name: &CStr, - sha256: &[u8; 32], - sizes: [usize; IMAGES], -) -> Result { - let fw = Firmware::request(name, dev).inspect_err(|e| { - dev_err!(dev, "board images {}: not available ({:?})\n", name, e) - })?; - let data = fw.data(); - let digest = glue::sha256(data); - if digest != *sha256 { - dev_err!( - dev, - "board images {}: sha256 {} is not the pinned one\n", - name, - Hex(&digest) - ); - return Err(EINVAL); - } - if data.len() < 16 || &data[..8] != MAGIC || le32(data, 8)? != VERSION { - dev_err!(dev, "board images {}: unknown header\n", name); - return Err(EINVAL); - } - if le32(data, 12)? as usize != IMAGES { - dev_err!(dev, "board images {}: not {} images\n", name, IMAGES); - return Err(EINVAL); - } - let mut at = 16 + IMAGES * 8; - let mut images: [Option>; IMAGES] = [None, None, None]; - for i in 0..IMAGES { - let kind = le32(data, 16 + i * 8)? as usize; - let size = le32(data, 16 + i * 8 + 4)? as usize; - if kind != i || size != sizes[i] { - dev_err!( - dev, - "board images {}: image {} is kind {} of {} bytes, need {}\n", - name, - i, - kind, - size, - sizes[i] - ); - return Err(EINVAL); - } - images[i] = Some(copy(data, at..at + size)?); - at += size; - } - if at != data.len() { - dev_err!(dev, "board images {}: trailing bytes\n", name); - return Err(EINVAL); - } - let [hardware, globals, power] = images; - dev_info!( - dev, - "board images {}: sha256 {} accepted\n", - name, - Hex(&digest) - ); - Ok(Images { - hardware: hardware.ok_or(EINVAL)?, - globals: globals.ok_or(EINVAL)?, - power: power.ok_or(EINVAL)?, - }) -} diff --git a/drivers/gpu/drm/asahi/g15/board/record.rs b/drivers/gpu/drm/asahi/g15/board/record.rs index 959bc6a4ab0a2..119a692064391 100644 --- a/drivers/gpu/drm/asahi/g15/board/record.rs +++ b/drivers/gpu/drm/asahi/g15/board/record.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! The J516C hardware record (0xb480 bytes) from the device tree. +//! The J516C hardware record (0xb480 bytes) from the device tree and the ADT. //! //! The record is HardwareDataB (identity, IOMapping records, the DVFS tables) at 0, HardwareDataA //! (the power, performance and thermal controllers) at [`HWDATA_A`], and the G15C tail (sensor @@ -13,12 +13,14 @@ //! state cap and base state; //! - a layout constant of the 2419 firmware's G15C record: flags, the IOMapping flag words, //! the identity words, fixed factors (65536, 95, 100, 1.02, 0.8/0.2, 625, 11000, 125, 5); -//! - a word the device tree does not carry ([`FROM_FILE`]), copied from the machine's board file -//! by [`from_file`] and listed with its rule in the project notes: the maximum power -//! (HardwareDataA +0x6a4; every other maximum-power word derives from it), the MTR sensor -//! masks, the relative-power table (the device tree's operating-point powers are a boot-loader -//! stand-in) and the tail's six sensor rows; -//! - a word the file carries and the firmware was shown not to need ([`UNNEEDED`]): zero. +//! - the tail's six sensor rows: entries of the ADT's `aon-ptd` sensor table ([`sensor_rows`]); +//! - a word neither carries ([`REFERENCE_WORDS`]): the value of the J516C reference machine +//! ([`Reference`]), as the T6030 templates carry their reference machine's: the maximum +//! power (HardwareDataA +0x6a4; every other maximum-power word derives from it), the MTR +//! sensor masks and the relative-power table (the device tree's operating-point powers are a +//! boot-loader stand-in); +//! - a word the reference machine's record carries and the firmware was shown not to need +//! ([`UNNEEDED`]): zero. //! //! HardwareDataA has the G15S shape of the T6030 templates with insertions ([`a`]): 8 bytes //! before the fast-die controller (+0x8bc), 16-wide instead of 8-wide coefficient rows (+0x200 @@ -152,14 +154,13 @@ const SE_KP1: usize = 40; const SE_RESET: usize = 41; const SE_TARGET: usize = 42; -/// One operating point: MHz, the clusters' voltages in mV (rounded up from microvolts), how -/// many voltage cells the node had, and mW. +/// One operating point: MHz, the clusters' voltages in mV (rounded up from microvolts) and how +/// many voltage cells the node had. #[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct Opp { pub(crate) mhz: u32, pub(crate) mv: [u32; CLUSTERS], pub(crate) cells: usize, - pub(crate) mw: u32, } impl Opp { @@ -198,40 +199,63 @@ pub(crate) struct Inputs<'a> { pub(crate) usc_base: u64, pub(crate) user_end: u64, pub(crate) timestamps: u64, - /// The maximum GPU power in mW ([`MAX_POWER`], a word the device tree does not carry). + /// The six sensor rows of the ADT's `aon-ptd` table ([`sensor_rows`]). + pub(crate) sensor_rows: [[u8; 12]; SENSOR_ROW_COUNT], + /// The words neither the device tree nor the ADT carries. + pub(crate) reference: &'a Reference, +} + +/// The record's words that neither the device tree nor the ADT carries, as one machine's +/// driver wrote them ([`REFERENCE_WORDS`]). +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Reference { + /// The maximum GPU power in mW ([`MAX_POWER`]). pub(crate) max_power_mw: u32, + /// The relative powers of DVFS states 1..7, percent of the top state's (state 8 is 100). + pub(crate) relative_power: [u32; RELATIVE_STATES], + /// The MTR fast-die sensor mask (written twice) and alarm mask, as the u64 at each place. + /// The device tree's `apple,fast-die0-sensor-mask` is another value. + pub(crate) fast_die_mask: u64, + pub(crate) alarm_mask: u64, } +/// DVFS states with a relative-power word of their own. +pub(crate) const RELATIVE_STATES: usize = 7; + +/// The J516C reference machine's words (its 2419 firmware driver's values on that machine). +/// Another J516C gets them as a J514S gets the T6030 templates' machine's: the operating +/// points, voltages, gains and sensor rows are its own, these are the reference machine's. +pub(crate) const J516C: Reference = Reference { + max_power_mw: 116739, + relative_power: [11, 23, 33, 44, 55, 71, 90], + fast_die_mask: 0x802a002b_802a002b, + alarm_mask: 0x902a102b_802a002b, +}; + /// HardwareDataA's maximum-power word (= `fw::initdata::hwdata::MAX_POWER_A`). pub(crate) const MAX_POWER: usize = HWDATA_A + 0x6a4; +const RELATIVE_POWER: usize = 0x1060; +const FAST_DIE_MASK: usize = HWDATA_A + 0x8ac; +const FAST_DIE_MASK_2: usize = HWDATA_A + 0x1490; +const ALARM_MASK: usize = HWDATA_A + 0x24a8; -/// Words the device tree does not carry, copied from the board file. Record offsets. +/// The record words [`Reference`] supplies. Record offsets. +#[cfg(test)] #[rustfmt::skip] -pub(crate) const FROM_FILE: [usize; 32] = [ - // The relative powers of states 1..7 (state 8 is 100): the device tree's operating-point - // powers are a boot-loader stand-in. +pub(crate) const REFERENCE_WORDS: [usize; 14] = [ 0x1060, 0x1064, 0x1068, 0x106c, 0x1070, 0x1074, 0x1078, - // HardwareDataA: the maximum power, the MTR fast-die masks (+0x8ac, +0x1490) and the alarm - // mask (+0x24a8); the device tree's apple,fast-die0-sensor-mask is another value. MAX_POWER, - HWDATA_A + 0x8ac, HWDATA_A + 0x8b0, - HWDATA_A + 0x1490, HWDATA_A + 0x1494, - HWDATA_A + 0x24a8, HWDATA_A + 0x24ac, - // The tail's six sensor rows (key, id, kind; 12 bytes each; the boot loader does not export - // the aon-ptd table). - HWDATA_A + 0x6c58, HWDATA_A + 0x6c5c, HWDATA_A + 0x6c60, - HWDATA_A + 0x6c64, HWDATA_A + 0x6c68, HWDATA_A + 0x6c6c, - HWDATA_A + 0x6c70, HWDATA_A + 0x6c74, HWDATA_A + 0x6c78, - HWDATA_A + 0x6c7c, HWDATA_A + 0x6c80, HWDATA_A + 0x6c84, - HWDATA_A + 0x6c88, HWDATA_A + 0x6c8c, HWDATA_A + 0x6c90, - HWDATA_A + 0x6c94, HWDATA_A + 0x6c98, HWDATA_A + 0x6c9c, + FAST_DIE_MASK, FAST_DIE_MASK + 4, + FAST_DIE_MASK_2, FAST_DIE_MASK_2 + 4, + ALARM_MASK, ALARM_MASK + 4, ]; -/// Words the board file carries that the record leaves zero: the firmware ran, passed every -/// check and reached its top state without them (one boot each). The cross-check accepts the -/// file's value there. 27 words in the HardwareDataB +0x38..+0x638 block and two all-ones words +/// Words the reference machine's record carries that this one leaves zero: the firmware ran, +/// passed every check and reached its top state without them (one boot each). 27 words in +/// the HardwareDataB +0x38..+0x638 block and two all-ones words /// after it (zero in the G15S templates); the tail's thirteen readings (a single at /// HardwareDataA +0x6d34, twelve singles after +0x6d80). +#[cfg(test)] #[rustfmt::skip] pub(crate) const UNNEEDED: [usize; 42] = [ 0xe0, 0xe8, 0xf4, 0xf8, 0xfc, 0x100, 0x104, 0x108, 0x10c, 0x2d8, 0x2dc, 0x2e0, 0x2e4, @@ -247,7 +271,7 @@ pub(crate) const UNNEEDED: [usize; 42] = [ /// rows whose kind byte is `g`, their presence mask, then the `r` and `f` rows' keys with a /// count of one each. const SENSOR_ROWS: usize = HWDATA_A + 0x6c58; -const SENSOR_ROW_COUNT: usize = 6; +pub(crate) const SENSOR_ROW_COUNT: usize = 6; const SENSOR_G_KEYS: usize = HWDATA_A + 0x6cdc; const SENSOR_G_MASK: usize = HWDATA_A + 0x6cfc; const SENSOR_R_KEY: usize = HWDATA_A + 0x6d04; @@ -492,15 +516,13 @@ fn hwdata_b(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { put32(out, 0x10dc + 4 * k, t.primary[k] as u32)?; put32(out, 0x111c + 4 * k, t.secondary[k] as u32)?; if k > 0 { - // Relative power (a FROM_FILE word on the J516C) and relative boost frequency. - if top.mw != 0 { - put32( - out, - 0x105c + 4 * k, - u32::try_from(100 * u64::from(ps.mw) / u64::from(top.mw)) - .map_err(|_| "a relative power does not fit its u32 word")?, - )?; - } + // Relative power (the reference machine's) and relative boost frequency. + let relative = match i.reference.relative_power.get(k - 1) { + Some(&r) if k < t.top() => r, + None if k == t.top() => 100, + _ => return Err("the reference relative powers are for another state count"), + }; + put32(out, RELATIVE_POWER - 4 + 4 * k, relative)?; if ps.mhz > base.mhz { let span = u64::from(top.mhz - base.mhz); put32( @@ -565,7 +587,7 @@ fn hwdata_a(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { |gain: usize| f32_mul(p[gain], period_s).ok_or("a gain is not a finite normal single"); let max_scaled = 100 * t.top() as u32; let base_scaled = mul(100, p[BASE_PSTATE])?; - let max_power = i.max_power_mw; + let max_power = i.reference.max_power_mw; let max_power_f = f32_from_u32(max_power); let mut w = |off: usize, v: u32| put32(out, a(off), v); w(0x4, p[AIC_CLKS])?; @@ -644,7 +666,7 @@ fn hwdata_a(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { w(0x864, max_scaled)?; w(0x86c, F_0_8)?; w(0x870, F_0_2)?; - // Fast-die controller (its sensor mask is a FROM_FILE word). + // Fast-die controller (its sensor mask follows below). w(0x8bc, 4)?; w(0x8d0, F_1)?; w(0x8d8, per_period(FD_KI)?)?; @@ -737,15 +759,46 @@ fn hwdata_a(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { w(SE + 0x11c, F_65536)?; w(SE + 0x12c, 40)?; w(SE + 0x130, max_scaled)?; - // Past the second sensor mask (FROM_FILE), the alarm mask (FROM_FILE) and the end flag. + // Past the second sensor mask, the alarm mask and the end flag. w(0x1290, 125)?; w(0x1294, 1)?; w(0x1aa0, 1)?; w(0x3aa4, F_5)?; + // The MTR sensor masks: the fast-die mask at both places, then the alarm mask. + put64(out, FAST_DIE_MASK, i.reference.fast_die_mask)?; + put64(out, FAST_DIE_MASK_2, i.reference.fast_die_mask)?; + put64(out, ALARM_MASK, i.reference.alarm_mask)?; Ok(()) } -/// The tail's sensor keys from the six rows (which [`from_file`] fills). +/// The six sensor rows of the ADT's `aon-ptd` `ptd_data_table`: 12-byte entries of a +/// little-endian four-character key (`RT0g` is stored as `"RT0g"`) and two words. The rows are +/// the `RT` entries of kind `g` (four, in table order), then the one of kind `r` and the one +/// of kind `f`; any other count is refused rather than guessed. +pub(crate) fn sensor_rows(table: &[u8]) -> R<[[u8; 12]; SENSOR_ROW_COUNT]> { + if table.is_empty() || table.len() % 12 != 0 { + return Err("the aon-ptd sensor table is not whole 12-byte entries"); + } + let mut rows = [[0u8; 12]; SENSOR_ROW_COUNT]; + let mut n = 0; + for (kind, wanted) in [(b'g', 4), (b'r', 1), (b'f', 1)] { + let mut found = 0; + for e in table.chunks_exact(12) { + if e[..2] == *b"RT" && e[3] == kind { + let row = rows.get_mut(n).ok_or("too many GPU sensor rows in aon-ptd")?; + row.copy_from_slice(e); + n += 1; + found += 1; + } + } + if found != wanted { + return Err("aon-ptd does not hold four g, one r and one f GPU sensor row"); + } + } + Ok(rows) +} + +/// The tail's sensor keys from the six rows. fn tail(out: &mut [u8]) -> R<()> { let mut g = 0usize; for row in 0..SENSOR_ROW_COUNT { @@ -775,16 +828,10 @@ fn tail(out: &mut [u8]) -> R<()> { Ok(()) } -/// Copies the [`FROM_FILE`] words of `file` (the board file's record) into `out`. -pub(crate) fn from_file(out: &mut [u8], file: &[u8]) -> R<()> { - for &at in FROM_FILE.iter() { - put32(out, at, get32(file, at)?)?; - } - Ok(()) -} - -/// The first word at which the built record differs from the file's, except the [`UNNEEDED`] -/// words, where the file's value is accepted as long as the built word is zero. +/// The first word at which a built record differs from a machine's own record (a host test's +/// fixture), except the [`UNNEEDED`] words, where the machine's value is accepted as long as +/// the built word is zero. +#[cfg(test)] pub(crate) fn cross_check(built: &[u8], file: &[u8]) -> Option<(usize, u32, u32)> { let word = |s: &[u8], at: usize| -> u32 { let mut w = [0u8; 4]; @@ -799,23 +846,20 @@ pub(crate) fn cross_check(built: &[u8], file: &[u8]) -> Option<(usize, u32, u32) .find(|&(at, b, f)| b != f && !(b == 0 && UNNEEDED.contains(&at))) } -/// Builds the record into `out` (zeroed first) from the device-tree inputs; the [`FROM_FILE`] -/// words must have been copied in first ([`from_file`]) or are left as they are. +/// Builds the record into `out` (zeroed first) from the inputs. pub(crate) fn build(out: &mut [u8], i: &Inputs<'_>) -> R<()> { if out.len() != SIZE { return Err("record of another size"); } - let mut keep = [0u32; FROM_FILE.len()]; - for (k, &at) in FROM_FILE.iter().enumerate() { - keep[k] = get32(out, at)?; - } out.fill(0); let t = tables(i.main)?; hwdata_b(out, i, &t)?; hwdata_a(out, i, &t)?; - // The kept words win over what the rules put there (the relative powers, the masks). - for (k, &at) in FROM_FILE.iter().enumerate() { - put32(out, at, keep[k])?; + for (k, row) in i.sensor_rows.iter().enumerate() { + let at = SENSOR_ROWS + 12 * k; + out.get_mut(at..at + 12) + .ok_or("record too short")? + .copy_from_slice(row); } tail(out) } @@ -828,18 +872,13 @@ mod tests { get32(b, o).unwrap() } - fn opp(mhz: u32, mv: [u32; 4], mw: u32) -> Opp { - Opp { - mhz, - mv, - cells: 4, - mw, - } + fn opp(mhz: u32, mv: [u32; 4]) -> Opp { + Opp { mhz, mv, cells: 4 } } /// Operating points shaped like the J516C's: eight voltages, the upper five with two states. fn main_opps() -> Vec { - let mut v = vec![opp(0, [125; 4], 0)]; + let mut v = vec![opp(0, [125; 4])]; let groups: [(u32, [u32; 4], &[u32]); 8] = [ (605, [605, 605, 615, 615], &[338]), (680, [680, 680, 665, 665], &[618]), @@ -852,7 +891,7 @@ mod tests { ]; for (_, mv, mhz) in groups { for &m in mhz { - v.push(opp(m, mv, m * 40)); + v.push(opp(m, mv)); } } v @@ -888,6 +927,27 @@ mod tests { p } + /// Arbitrary reference words, distinct from the J516C's. + const TEST_REFERENCE: Reference = Reference { + max_power_mw: 54321, + relative_power: [0x1000, 0x1001, 0x1002, 0x1003, 0x1004, 0x1005, 0x1006], + fast_die_mask: 0x1111_2222_3333_4444, + alarm_mask: 0x5555_6666_7777_8888, + }; + + /// A sensor row: the key's characters in storage order, then two words. + fn row(key: &[u8; 4], id: u32, kind: u32) -> [u8; 12] { + let mut r = [0u8; 12]; + r[..4].copy_from_slice(key); + r[4..8].copy_from_slice(&id.to_le_bytes()); + r[8..].copy_from_slice(&kind.to_le_bytes()); + r + } + + fn rows() -> [[u8; 12]; SENSOR_ROW_COUNT] { + [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"].map(|k| row(k, 0x10001, 0x1401)) + } + fn inputs<'a>(main: &'a [Opp], cs: &'a [Opp], slots: &'a [IoSlot]) -> Inputs<'a> { Inputs { props: props(), @@ -901,7 +961,8 @@ mod tests { usc_base: 0x10_0000_0000, user_end: 0x2ff_ffff_8000, timestamps: 0xfffffc2ffc000000, - max_power_mw: 54321, + sensor_rows: rows(), + reference: &TEST_REFERENCE, } } @@ -927,9 +988,12 @@ mod tests { #[test] fn unneeded() { - // No UNNEEDED word is a FROM_FILE word; the cross-check accepts the file's value there - // only against a zero built word, and nothing else. - assert!(UNNEEDED.iter().all(|at| !FROM_FILE.contains(at))); + // No UNNEEDED word is a reference or sensor-row word; the cross-check accepts the + // file's value there only against a zero built word, and nothing else. + let rows = SENSOR_ROWS..SENSOR_ROWS + 12 * SENSOR_ROW_COUNT; + assert!(UNNEEDED + .iter() + .all(|at| !REFERENCE_WORDS.contains(at) && !rows.contains(at))); assert!(UNNEEDED.iter().all(|&at| at % 4 == 0 && at + 4 <= SIZE)); let mut file = vec![0u8; SIZE]; for (k, &at) in UNNEEDED.iter().enumerate() { @@ -957,9 +1021,10 @@ mod tests { assert_eq!(a(0x1aa0), HWDATA_A + 0x24b8); assert_eq!(a(0x3aa4), HWDATA_A + 0x64bc); assert_eq!(MAX_POWER, a(0x6a4)); - // FROM_FILE is sorted, unique and inside the record. - assert!(FROM_FILE.windows(2).all(|w| w[0] < w[1])); - assert!(FROM_FILE.iter().all(|&o| o % 4 == 0 && o + 4 <= SIZE)); + // REFERENCE_WORDS is sorted, unique and inside the record. + assert!(REFERENCE_WORDS.windows(2).all(|w| w[0] < w[1])); + assert!(REFERENCE_WORDS.iter().all(|&o| o % 4 == 0 && o + 4 <= SIZE)); + assert!(SENSOR_ROWS + 12 * SENSOR_ROW_COUNT <= SENSOR_G_KEYS); } #[test] @@ -995,7 +1060,6 @@ mod tests { mhz, mv: [mv, 0, 0, 0], cells: 1, - mw: 0, }) .collect(); let slots = [ @@ -1017,20 +1081,23 @@ mod tests { }, ]; let i = inputs(&main, &cs, &slots); - let mut r = vec![0u8; SIZE]; - // FROM_FILE words survive the build; the sensor rows feed the tail. - for (k, &at) in FROM_FILE.iter().enumerate() { - put32(&mut r, at, 0x1000 + k as u32).unwrap(); - } - // The file's maximum power is what the input repeats. - put32(&mut r, MAX_POWER, 54321).unwrap(); - let rows = [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"]; - for (k, row) in rows.iter().enumerate() { - put32(&mut r, SENSOR_ROWS + 12 * k, u32::from_le_bytes(**row)).unwrap(); - } + // Whatever the buffer held before is not part of the record. + let mut r = vec![0x5au8; SIZE]; build(&mut r, &i).unwrap(); let good = r.clone(); - assert_eq!(get(&r, 0x1060), 0x1000); + // The reference words: relative powers of states 1..7, the maximum power, the masks. + for k in 0..RELATIVE_STATES { + assert_eq!(get(&r, 0x1060 + 4 * k), 0x1000 + k as u32); + } + assert_eq!(get(&r, FAST_DIE_MASK), 0x3333_4444); + assert_eq!(get(&r, FAST_DIE_MASK + 4), 0x1111_2222); + assert_eq!(get(&r, FAST_DIE_MASK_2), 0x3333_4444); + assert_eq!(get(&r, FAST_DIE_MASK_2 + 4), 0x1111_2222); + assert_eq!(get(&r, ALARM_MASK), 0x7777_8888); + assert_eq!(get(&r, ALARM_MASK + 4), 0x5555_6666); + // The sensor rows as given. + assert_eq!(&r[SENSOR_ROWS..SENSOR_ROWS + 12], &rows()[0]); + assert_eq!(&r[SENSOR_ROWS + 60..SENSOR_ROWS + 72], &rows()[5]); assert_eq!(get(&r, 0x30f0), 0); // Identity, IOMapping records, flags. assert_eq!(get(&r, 0xa28), 0x6031); @@ -1099,8 +1166,15 @@ mod tests { let mut j = inputs(&main, &cs, &slots); j.empty_writable = &[0]; assert!(build(&mut r, &j).is_err()); + let zero = Reference { + max_power_mw: 0, + ..TEST_REFERENCE + }; + let mut j = inputs(&main, &cs, &slots); + j.reference = &zero; + assert!(build(&mut r, &j).is_err()); let mut j = inputs(&main, &cs, &slots); - j.max_power_mw = 0; + j.sensor_rows[0] = row(b"RT0x", 1, 1); assert!(build(&mut r, &j).is_err()); let mut j = inputs(&main, &cs, &slots); j.zones = &[100, 101, 1]; @@ -1127,34 +1201,49 @@ mod tests { let mut j = inputs(&main, &cs, &slots); j.zones = &[5000, 200, 0x4000_0000]; assert_eq!(refused(&j), Err(OVERFLOW)); - let mut skewed = main.clone(); - skewed[1].mw = u32::MAX; - skewed[13].mw = 1; + // The seven relative powers fit eight states above state 0 and nothing else. + let fewer = &main[..PERF_STATES - 2]; + assert_eq!(tables(fewer).unwrap().top(), 7); assert_eq!( - refused(&inputs(&skewed, &cs, &slots)), - Err("a relative power does not fit its u32 word") + refused(&inputs(fewer, &cs, &slots)), + Err("the reference relative powers are for another state count") ); - // The admission cross-check as initdata::blob() composes it: the FROM_FILE words from - // the file, the rest built, then the first differing word against the file. A file the - // inputs reproduce passes; a rule word the file holds differently is found at its - // offset; a FROM_FILE word follows the file and is never a difference. - let first_difference = super::super::words::first_difference; - let checked = |file: &[u8]| { - let mut built = vec![0u8; SIZE]; - from_file(&mut built, file).unwrap(); - build(&mut built, &i).unwrap(); - (first_difference(&built, file), built) - }; - assert_eq!(checked(&good).0, None); - let mut file = good.clone(); - put32(&mut file, a(0x4c), 801).unwrap(); - assert_eq!(checked(&file).0, Some((a(0x4c), 800, 801))); - let mut file = good.clone(); - assert!(FROM_FILE.contains(&0x1064)); - put32(&mut file, 0x1064, get(&good, 0x1064) ^ 0x5a).unwrap(); - let (difference, built) = checked(&file); - assert_eq!(difference, None); - assert_eq!(get(&built, 0x1064), get(&good, 0x1064) ^ 0x5a); + } + + #[test] + fn sensor_rows_from_the_ptd_table() { + // A table shaped like the J516C's: other sensors around the GPU's, the f row first. + let mut table = Vec::new(); + for r in [ + row(b"YD0C", 0x5e0001, 0x1040405), + row(b"RT0a", 0x6b0001, 0x13), + row(b"RT0f", 0xcb0001, 0x1601), + row(b"RE0g", 0xaa0001, 0xff01), + row(b"RT0g", 0xab0001, 0x1401), + row(b"RT1g", 0x8d0001, 0x1401), + row(b"RT2g", 0x8b0001, 0x1401), + row(b"RT3g", 0xad0001, 0x1401), + row(b"RT0p", 0x6d0001, 0x13), + row(b"RT0r", 0xeb0001, 0x1401), + row(b"XEXE", 0x3c0001, 0xff00), + ] { + table.extend_from_slice(&r); + } + let got = sensor_rows(&table).unwrap(); + let keys: Vec<&[u8]> = got.iter().map(|r| &r[..4]).collect(); + assert_eq!(keys, [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"]); + assert_eq!(got[5], row(b"RT0f", 0xcb0001, 0x1601)); + assert_eq!(got[4], row(b"RT0r", 0xeb0001, 0x1401)); + // Refused: no table, a partial entry, a missing row, a fifth g row, a second f row. + assert!(sensor_rows(&[]).is_err()); + assert!(sensor_rows(&table[..table.len() - 1]).is_err()); + assert!(sensor_rows(&table[12..table.len() - 12]).is_ok()); + assert!(sensor_rows(&table[..9 * 12]).is_err()); + for extra in [row(b"RT4g", 1, 1), row(b"RT1f", 1, 1)] { + let mut t = table.clone(); + t.extend_from_slice(&extra); + assert!(sensor_rows(&t).is_err()); + } } // --- The fixture: a saved copy of a J516C's device tree and its private board file. --- @@ -1236,14 +1325,12 @@ mod tests { for (c, m) in mv.iter_mut().enumerate().take(cells) { *m = be32(uv, 4 * c).div_ceil(1000); } - let mw = prop(props, "opp-microwatt").map_or(0, |w| be32(w, 0) / 1000); ( p.clone(), Opp { mhz: (hz / 1_000_000) as u32, mv, cells, - mw, }, ) }) @@ -1262,16 +1349,8 @@ mod tests { mhz: 24, mv: [635, 0, 0, 0], cells: 1, - mw: 0, }]; let mut seed = vec![0u8; SIZE]; - put32(&mut seed, MAX_POWER, 54321).unwrap(); - for (k, key) in [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"] - .iter() - .enumerate() - { - put32(&mut seed, SENSOR_ROWS + 12 * k, u32::from_le_bytes(**key)).unwrap(); - } assert_eq!(build(&mut seed, &inputs(&main, &cs, &[])), Ok(())); for prop in 0..PROPS.len() { for value in [ @@ -1289,16 +1368,59 @@ mod tests { } } - /// Builder on the saved device tree + the file's FROM_FILE words == the file's hardware - /// record, byte for byte. Needs `G15_J516C_FIXTURES=` with `j516c.dtb` and - /// `t6031-j516c-board.bin` (off the tree): ignored by default, `--ignored` runs it. + /// A property of an ADT node found by its path of `name`s from the root: nodes of u32 + /// property and child counts, properties of a 32-byte name, a u32 size (top bit a flag) + /// and the value padded to 4 bytes. + fn adt_prop<'a>(adt: &'a [u8], path: &[&str], name: &str) -> Option<&'a [u8]> { + fn le(b: &[u8], o: usize) -> usize { + u32::from_le_bytes(b[o..o + 4].try_into().unwrap()) as usize + } + // The node at `at`: its properties, its children's offsets and the offset past it. + fn node(adt: &[u8], at: usize) -> (Vec<(&[u8], &[u8])>, Vec, usize) { + let (np, nc) = (le(adt, at), le(adt, at + 4)); + let mut o = at + 8; + let mut props = Vec::new(); + for _ in 0..np { + let n = &adt[o..o + 32]; + let n = &n[..n.iter().position(|&b| b == 0).unwrap_or(32)]; + let size = le(adt, o + 32) & 0x7fff_ffff; + props.push((n, &adt[o + 36..o + 36 + size])); + o += 36 + size.next_multiple_of(4); + } + let mut children = Vec::new(); + for _ in 0..nc { + children.push(o); + o = node(adt, o).2; + } + (props, children, o) + } + let mut at = 0; + for want in path { + let (_, children, _) = node(adt, at); + let named = |c: usize| { + let (props, _, _) = node(adt, c); + props.iter().any(|(n, v)| { + *n == b"name" && v.split(|&b| b == 0).next() == Some(want.as_bytes()) + }) + }; + at = *children.iter().find(|&&c| named(c))?; + } + node(adt, at).0.into_iter().find(|(n, _)| *n == name.as_bytes()).map(|(_, v)| v) + } + + /// Builder on the saved device tree and ADT with the J516C reference words == the J516C + /// reference machine's own hardware record, byte for byte except the [`UNNEEDED`] words. + /// Needs `G15_J516C_FIXTURES=` with `j516c.dtb` (`/sys/firmware/fdt`), `adt.bin` (the + /// boot loader's ADT) and `t6031-j516c-board.bin` (that machine's record), all off the + /// tree: ignored by default, `--ignored` runs it. #[test] #[ignore = "needs the off-tree J516C fixtures (G15_J516C_FIXTURES)"] fn j516c_fixture() { let dir = std::env::var_os("G15_J516C_FIXTURES") - .expect("G15_J516C_FIXTURES names the directory with j516c.dtb and the board file"); + .expect("G15_J516C_FIXTURES names the directory with j516c.dtb, adt.bin, the record"); let dir = std::path::Path::new(&dir); let dtb = std::fs::read(dir.join("j516c.dtb")).unwrap(); + let adt = std::fs::read(dir.join("adt.bin")).unwrap(); let blob = std::fs::read(dir.join("t6031-j516c-board.bin")).unwrap(); assert_eq!(&blob[..8], b"T6031BRD"); let at = 16 + 3 * 8; @@ -1389,10 +1511,13 @@ mod tests { usc_base: 0x10_0000_0000, user_end: 0x2ff_ffff_8000, timestamps: 0xfffffc2ffc000000, - max_power_mw: get(file, MAX_POWER), + sensor_rows: sensor_rows( + adt_prop(&adt, &["arm-io", "aon-ptd"], "ptd_data_table").expect("aon-ptd table"), + ) + .unwrap(), + reference: &J516C, }; let mut built = vec![0u8; SIZE]; - from_file(&mut built, file).unwrap(); build(&mut built, &i).unwrap(); assert_eq!(cross_check(&built, file), None, "(offset, built, file)"); for &at in UNNEEDED.iter() { diff --git a/drivers/gpu/drm/asahi/g15/board/words.rs b/drivers/gpu/drm/asahi/g15/board/words.rs index 4c5082465d428..f879d9b671672 100644 --- a/drivers/gpu/drm/asahi/g15/board/words.rs +++ b/drivers/gpu/drm/asahi/g15/board/words.rs @@ -235,6 +235,7 @@ pub(crate) fn power( /// The first word (4-byte aligned offset) at which `a` and `b` differ, with both values; a /// missing word reads 0. `None` when they are equal. +#[cfg(test)] pub(crate) fn first_difference(a: &[u8], b: &[u8]) -> Option<(usize, u32, u32)> { let word = |s: &[u8], at: usize| -> u32 { let mut w = [0u8; 4]; @@ -584,7 +585,7 @@ mod tests { /// Builder on the saved device tree == the private file's Globals and Power, byte for byte. /// Needs `G15_J516C_FIXTURES=` with `j516c.dtb` (`/sys/firmware/fdt` of the machine) - /// and `t6031-j516c-board.bin` (its board file); both stay off the tree, so the test is + /// and `t6031-j516c-board.bin` (its own images); both stay off the tree, so the test is /// ignored by default (`--ignored` runs it) and fails, rather than passes, without them. #[test] #[ignore = "needs the off-tree J516C fixtures (G15_J516C_FIXTURES)"] @@ -595,7 +596,7 @@ mod tests { let dtb = std::fs::read(dir.join("j516c.dtb")).unwrap(); let blob = std::fs::read(dir.join("t6031-j516c-board.bin")).unwrap(); - // The board file: magic, version, count, 3 x (kind, size), images (board/blob.rs). + // The machine's images: magic, version, count, 3 x (kind, size), then the images. assert_eq!(&blob[..8], b"T6031BRD"); assert_eq!(get32(&blob, 8), 1); assert_eq!(get32(&blob, 12), 3); diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index 7a06aa64ffbc3..70decd06fca0d 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -61,8 +61,10 @@ pub(crate) mod glue { fn asahi_g15_machine(compatible: *const core::ffi::c_char) -> bool; fn asahi_g15_hold_power(dev: *mut core::ffi::c_void); fn asahi_g15_release_power(dev: *mut core::ffi::c_void); - fn asahi_g15_module_get(); + fn asahi_g15_module_try_get() -> bool; + fn asahi_g15_module_put(); pub(crate) fn asahi_g15_reprobe(); + pub(crate) fn asahi_g15_report_loader_gpu(); fn asahi_g15_adt_phram(start: *mut u64, size: *mut u64) -> core::ffi::c_int; fn asahi_g15_ram_disjoint(start: u64, size: u64) -> bool; fn asahi_g15_ram_within(start: u64, size: u64) -> bool; @@ -70,7 +72,9 @@ pub(crate) mod glue { fn asahi_g15_ioremap_wc(start: u64, size: usize) -> *mut core::ffi::c_void; fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; pub(crate) fn asahi_g15_pmp_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + pub(crate) fn asahi_g15_display_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_get(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + pub(crate) fn asahi_g15_pmp_ready() -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_put(dev: *mut core::ffi::c_void); fn asahi_g15_reserved_uat(index: u32, start: u64, size: u64) -> bool; } @@ -120,12 +124,17 @@ pub(crate) mod glue { RETAINED.load(Ordering::Acquire) } - /// Pins this module until reboot, once (`__module_get(THIS_MODULE)`). - pub(crate) fn retain_module() { - if !RETAINED.swap(true, Ordering::AcqRel) { - // SAFETY: The module is live while its code runs; the reference is never put. - unsafe { asahi_g15_module_get() }; - } + /// Pins this module until reboot, once (`try_module_get(THIS_MODULE)`, see `retain.rs`). + /// ENODEV, with nothing pinned or marked, once the module's removal has begun. + pub(crate) fn retain_module() -> Result { + crate::g15::retain::retain( + &RETAINED, + // SAFETY: Takes a reference to this module unless its removal has begun. + || unsafe { asahi_g15_module_try_get() }, + // SAFETY: Drops only the reference this call took; the winner's is never put. + || unsafe { asahi_g15_module_put() }, + ) + .map_err(|_| ENODEV) } /// The phram "adt" export: start and size. @@ -478,6 +487,17 @@ pub(crate) fn admit( } else if !glue::machine(cfg.machine) { dev_err!(dev, "G15: not a {} machine\n", cfg.name); return Err(ENODEV); + } else { + // No PMP vote here, but the first start follows the native display all the same. + // SAFETY: `dev` is the bound GPU device. + let ret = unsafe { glue::asahi_g15_display_prepare(dev.as_raw().cast()) }; + if ret != 0 { + let err = Error::from_errno(ret); + if err != EPROBE_DEFER { + dev_err!(dev, "native display not admitted: {:?}\n", err); + } + return Err(err); + } } let adt = adt::read(dev, cfg) .inspect_err(|e| dev_err!(dev, "GPU regions not admitted: {:?}\n", e))?; @@ -674,8 +694,16 @@ pub(crate) fn start( return Err(e); } + // Pin the module before anything firmware-visible. Once its removal has begun this fails + // and the firmware is left untouched, cleaned up as for a failed self-check: the vote is + // released and the session stays unretained. + if let Err(e) = glue::retain_module() { + dev_err!(d, "G15: the module is being removed; the firmware is not started\n"); + // Mapped with cached firmware classes: never unmapped. + core::mem::forget(objects); + return Err(e); + } // From here on the firmware is touched: nothing it can reach is freed, the vote included. - glue::retain_module(); vote.keep(); let rtkit = match Rtkit::new(&d, shared) { Ok(rtkit) => rtkit, diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 6fc1e03fb637c..66e52e6981197 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -146,6 +146,11 @@ pub(crate) struct Context<'a> { pub(crate) usc_base: u64, } +struct CheckedContext<'a> { + vm: &'a mut mmu::VmReadGuard, + usc_base: u64, +} + /// A stream whose timestamps and ordinals are resolved ([`prepare`]), before its spans are /// checked ([`Prepared::validate`]). pub(crate) struct Prepared<'a> { @@ -377,7 +382,13 @@ impl Timestamps<'_> { /// Checks that `[start, end)` (rounded out to pages) is user memory mapped with the requested /// GPU access: a driver range ENOENT, outside the user range EINVAL, an unmapped page ENOENT, a /// page without the access EACCES. -pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) -> Result { +pub(crate) fn cover( + vm: &mut mmu::VmReadGuard, + start: u64, + end: u64, + read: bool, + write: bool, +) -> Result { let mask = mmu::UAT_PGMSK as u64; if end <= start { return Err(EINVAL); @@ -407,7 +418,11 @@ pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) } /// Attachment spans, 128-byte granular, actually mapped with `write` or read access. -fn check_attachments(vm: &mmu::Vm, list: &[uapi::drm_asahi_attachment], write: bool) -> Result { +fn check_attachments( + vm: &mut mmu::VmReadGuard, + list: &[uapi::drm_asahi_attachment], + write: bool, +) -> Result { for attachment in list { let end = attachment .pointer @@ -467,8 +482,8 @@ fn geometry_bytes(cmd: &uapi::drm_asahi_cmd_render) -> (u64, u64) { } /// Per-command render admission. -fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { - let vm = context.vm; +fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { + let vm = &mut *context.vm; let allowed = PROCESS_EMPTY_TILES | NO_VERTEX_CLUSTERING | RSRC_SPEC_HI @@ -552,11 +567,11 @@ fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Resu /// Per-command compute admission. fn check_compute( - context: &Context<'_>, + context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_compute, attachments: &[uapi::drm_asahi_attachment], ) -> Result { - let vm = context.vm; + let vm = &mut *context.vm; let (base, end) = (cmd.cdm_ctrl_stream_base, cmd.cdm_ctrl_stream_end); if cmd.flags != 0 || cmd.sampler_count != 0 @@ -650,6 +665,12 @@ impl Prepared<'_> { counts, timestamps, } = self; + // On error, unlock before the last timestamp mapping can drop and unmap. + let mut access = context.vm.read_lock()?; + let mut context = CheckedContext { + vm: &mut access, + usc_base: context.usc_base, + }; let mut commands = KVec::with_capacity(resolved.len(), GFP_KERNEL)?; let mut compute = KVec::new(); let mut index = 0; @@ -658,12 +679,12 @@ impl Prepared<'_> { match entry.kind { SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS => { // Validated readable, never serialized on G15. - check_attachments(context.vm, &attachment_list(entry.payload)?, false)?; + check_attachments(&mut *context.vm, &attachment_list(entry.payload)?, false)?; } SET_COMPUTE_ATTACHMENTS => compute = attachment_list(entry.payload)?, CMD_RENDER => { let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; - check_render(context, &cmd)?; + check_render(&mut context, &cmd)?; commands.push( Command::Render(KBox::new( RenderCommand { @@ -678,7 +699,7 @@ impl Prepared<'_> { } CMD_COMPUTE => { let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; - check_compute(context, &cmd, &compute)?; + check_compute(&mut context, &cmd, &compute)?; let slots = resolved.get(index).ok_or(EINVAL)?; commands.push( Command::Compute(KBox::new( diff --git a/drivers/gpu/drm/asahi/g15/encode.rs b/drivers/gpu/drm/asahi/g15/encode.rs index e46ecd05bdf78..a62cca8613b33 100644 --- a/drivers/gpu/drm/asahi/g15/encode.rs +++ b/drivers/gpu/drm/asahi/g15/encode.rs @@ -64,12 +64,23 @@ const fn align(n: u64, a: u64) -> u64 { } /// Firmware stamp value of ordinal `n` on the queue with this seed. -/// Stamps step by 0x100 through 0xffffff values and never read 0. +/// Stamps step by 0x100, including zero when the counter wraps. pub(crate) const fn stamp_seed(seed: u32, n: u64) -> u32 { - let base = (seed / 0x100) as u64; - ((((base - 1 + n % 0xffffff) % 0xffffff) + 1) * 0x100) as u32 + seed.wrapping_add((n as u32).wrapping_mul(0x100)) } +/// Ordinals after which [`stamp_seed`] repeats: 2^32 / 0x100. +pub(crate) const STAMP_PERIOD: u64 = 1 << 24; + +/// Effective render ordinal at which the TA stamp wraps (0xffffff00 -> 0): 0x860000 with the TA +/// seed. A wrap that skipped zero stopped the firmware ("DAG: Non Sequential Stamp Updates"); +/// the renumbering keeps the stamps away from it. The FS stamp wraps later (0xc30000). +pub(crate) const RENDER_WRAP: u64 = 0xffffff - (hw::engine::TA_SEED / 0x100) as u64 + 1; + +/// Compute ordinal at which the compute stamp wraps (0xffffff00 -> 0): 0x3f0000 (4128768 +/// computes) with the compute seed, where the firmware may stop as at the TA stamp wrap. +pub(crate) const COMPUTE_WRAP: u64 = STAMP_PERIOD - (hw::engine::COMPUTE_SEED / 0x100) as u64; + /// Compute stamp of ordinal `n`. pub(crate) const fn stamp(n: u64) -> u32 { stamp_seed(hw::engine::COMPUTE_SEED, n) @@ -102,8 +113,9 @@ pub(crate) struct ComputeRefs { pub(crate) scratch: [u64; 5], } -/// Encodes compute command `c` as ordinal `n`. The engine copies the result into the slot -/// after zeroing the slot's images. +/// Encodes compute command `c` as ordinal `n` with stamp `t` (the engine's renumbering moves +/// the stamp off [`stamp`]`(n)`). The engine copies the result into the slot after zeroing the +/// slot's images. #[allow(clippy::too_many_arguments)] pub(crate) fn encode_compute( r: &ComputeRefs, @@ -113,12 +125,12 @@ pub(crate) fn encode_compute( ts_va: &[u64; 2], ctx: u32, n: u64, + t: u32, fault_mask: u32, w: &mut [u8; compute::WORK_SIZE], q: &mut [u8; compute::SEQ_SIZE], ) { let (wv, qv, h, pre) = (r.work, r.qinfo, r.hwdata, r.preempt_gva); - let t = stamp(n); let att_count = att.len().min(hw::sizes::MAX_ATTACHMENTS); // The saved-state pointers into the preemption buffer and the two 0x1c words; a SoC // encoded "fresh" carries zero in all six, as a first command does. diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 51b3cfd10d594..1fa5c60bd1122 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -155,10 +155,10 @@ pub(crate) struct Flight { pub(crate) fs_start: u32, /// Channel producers (TA, FS, compute) after the prefix's records. pub(crate) rec: [u32; 3], - /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none): + /// Compute member j's FW stamp target, and the FS target it waits on directly: /// its predecessors in the prefix (for the compute-hang check). pub(crate) cstamp: [u32; sizes::COMPUTE_SLOTS], - pub(crate) cwait: [u32; sizes::COMPUTE_SLOTS], + pub(crate) cwait: [Option; sizes::COMPUTE_SLOTS], /// Compute slots, pass slots and clustering buffers it holds. pub(crate) claims: Claims, /// The prefix rebuilds the render allocators (priming); nothing is published behind it. @@ -189,7 +189,7 @@ impl Flight { fs_start: 0, rec: [0; 3], cstamp: [0; sizes::COMPUTE_SLOTS], - cwait: [0; sizes::COMPUTE_SLOTS], + cwait: [None; sizes::COMPUTE_SLOTS], claims: Claims::new(), rebuild: false, doomed: false, @@ -364,6 +364,12 @@ pub(crate) struct Engine { /// Host compute ordinal and draw ordinal of the last publication. pub(crate) ordinal: u64, pub(crate) draw: u64, + /// Added to the draw ordinal before computing the TA/FS stamps; moved by + /// `renumber_render_stamps()`. + pub(crate) stamp_base: u64, + /// Added to the compute ordinal before computing the compute stamp; moved by + /// `renumber_compute_stamps()`. + pub(crate) compute_stamp_base: u64, /// Newest published receipts, the tail. pub(crate) target: u32, pub(crate) head: u32, @@ -385,9 +391,11 @@ pub(crate) struct Engine { pub(crate) rebuild_live: bool, /// A ready prefix waits only for a record's consumption: repoll sooner. pub(crate) chan_wait: bool, + /// A ready job waits for an ASID held by a completing failed job. + pub(crate) slot_wait: bool, /// Progress tracking of the oldest prefix. pub(crate) progress_flight: u64, - pub(crate) progress: [u32; 3], + pub(crate) progress: [Option; 3], pub(crate) stats: Stats, /// Phase timers (debug_flags bit 49). pub(crate) prof: Prof, @@ -409,6 +417,8 @@ impl Engine { shared: None, ordinal: 0, draw: 0, + stamp_base: 0, + compute_stamp_base: 0, target: 0, head: 0, ta_target: 0, @@ -424,8 +434,9 @@ impl Engine { render_registered: false, rebuild_live: false, chan_wait: false, + slot_wait: false, progress_flight: 0, - progress: [0; 3], + progress: [None; 3], stats: Stats::default(), prof: Prof::default(), debugfs: None, @@ -515,8 +526,12 @@ impl Firmware { if self.engine.flights.capacity() < sizes::FLIGHTS { self.engine.flights.reserve(sizes::FLIGHTS, GFP_KERNEL)?; } + // A job arriving at an idle engine: the PB reclaim, if due, happens before its + // publication (idle polls are not re-armed, so this is where a long idle ends). + self.reclaim_check(); self.engine.pending.push(job.clone(), GFP_KERNEL)?; job.progress.set_pending(true); + self.note_idle(); let q = &job.queue; let owned = self.engine.queue_published(&job) + self @@ -542,6 +557,7 @@ impl Firmware { let mut result = Ok(()); self.engine.chan_wait = false; + self.engine.slot_wait = false; // Nothing is published while a hang request is armed. while self.recovery.failed().is_none() && !self.engine.frozen @@ -552,12 +568,17 @@ impl Firmware { break; }; let job = self.engine.pending[index].clone(); + // Inside a renumbering margin nothing is published behind a flight: the flights + // drain and the next publication renumbers the stamps while idle (once every ~8.7M + // renders or ~4.0M computes with margins of 100000). + let renumber_due = self.renumber_due(); if let Some(tail) = self.engine.flights.last() { if self.engine.flights.len() >= max_inflight || !Arc::ptr_eq(&job.vm, &tail.job.vm) || self.recovery.reinit || self.recovery.fs_resync || self.engine.rebuild_live + || renumber_due { self.engine.stats.hold[0] += (self.engine.flights.len() >= max_inflight) as u64; break; @@ -588,7 +609,7 @@ impl Firmware { dev_err!( self.graph.dev.as_ref(), "G15: publication of ctx {} failed after a queue WRITE ({:?})\n", - job.vm.slot, + job.slot(), e ); complete(done, &job, Err(EIO)); @@ -601,6 +622,9 @@ impl Firmware { } } } + // A publication that left nothing published and nothing pending (jobs that failed + // before any visible record) starts the idle. + self.note_idle(); result } @@ -668,6 +692,15 @@ impl Firmware { /// yet (no free slot, or a channel record still unconsumed); EIO once a queue WRITE /// advanced; any other error means nothing was published. fn publish_prefix(&mut self, job: &Arc, done: &mut Completions) -> Result { + if let Err(error) = job.bind() { + if error == ENOSPC { + // A failed job can keep its binding until scheduler references drop after + // this pass. Leave healthy work pending and retry without the device lock. + self.engine.slot_wait = true; + return Err(EAGAIN); + } + return Err(error); + } let mut f = Flight::new(job.clone()); let result = self.publish_prefix_into(job, &mut f); match result { @@ -675,7 +708,11 @@ impl Firmware { // Never published: its storage is free again, and horizon jobs taken into it // fail with it. self.slots.release(&mut f.claims); + if job.progress.next() == job.progress.sent() { + job.release_binding(); + } for j2 in f.group.iter() { + j2.release_binding(); complete(done, j2, Err(e)); } } @@ -684,6 +721,94 @@ impl Firmware { result } + /// The TA stamp is inside its renumbering margin. + fn render_renumber_due(&self) -> bool { + let margin = params::stamp_reseed_margin(); + let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD; + margin != 0 && ordinal + margin >= encode::RENDER_WRAP + } + + /// The compute stamp is inside its renumbering margin. + fn compute_renumber_due(&self) -> bool { + let margin = params::compute_reseed_margin(); + let ordinal = (self.engine.ordinal + self.engine.compute_stamp_base) % encode::STAMP_PERIOD; + margin != 0 && ordinal + margin >= encode::COMPUTE_WRAP + } + + /// Either stamp is inside its renumbering margin. + fn renumber_due(&self) -> bool { + self.render_renumber_due() || self.compute_renumber_due() + } + + /// Computes and renders the current numbering has left before its compute and TA stamps + /// wrap (0 once wrapped), for `asahi-g15/stats`. + pub(crate) fn stamps_left(&self) -> (u64, u64) { + let period = encode::STAMP_PERIOD; + let c = (self.engine.ordinal + self.engine.compute_stamp_base) % period; + let r = (self.engine.draw + self.engine.stamp_base) % period; + ( + encode::COMPUTE_WRAP.saturating_sub(c), + encode::RENDER_WRAP.saturating_sub(r), + ) + } + + /// Renumbers the TA/FS stamps from their seeds before the TA stamp wraps, after 0x860000 + /// renders (about six hours of desktop), where the firmware may stop. Only called with + /// nothing in flight: the last flight retired with every stamp equal to its target, so both + /// queues are idle and the next render waits on exactly the seeds written here. + fn renumber_render_stamps(&mut self) -> Result { + if !self.render_renumber_due() { + return Ok(()); + } + let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD; + let r = &self.graph.render; + for (e, seed) in [ids::TA_SEED, ids::FS_SEED].into_iter().enumerate() { + r.user_stamp[e].w32(0, seed)?; + r.fw_stamp[e].w32(0, seed)?; + } + // The next render (draw + 1) counts as ordinal 1 again: modulo the period of + // `stamp_seed`, whose ordinal 0 is the seeds just written. + let period = encode::STAMP_PERIOD; + self.engine.stamp_base = (period - self.engine.draw % period) % period; + self.engine.ta_target = ids::TA_SEED; + self.engine.fs_target = ids::FS_SEED; + self.engine.tail_fs = ids::FS_SEED; + dev_info!( + self.graph.dev.as_ref(), + "G15: TA/FS stamps renumbered from the seeds at draw {} (ordinal {:#x})\n", + self.engine.draw, + ordinal + ); + Ok(()) + } + + /// Renumbers the compute stamp from its seed before it wraps, after 0x3f0000 computes, + /// where the firmware may stop. Like the TA/FS renumbering only with nothing in flight: the + /// compute queue is idle with both stamps at its target. The stamps are written back as + /// `ComputeGraph::init()` leaves them with no stamp base, so the next compute is ordinal 1 + /// on the stamps of a fresh boot. + fn renumber_compute_stamps(&mut self) -> Result { + if !self.compute_renumber_due() { + return Ok(()); + } + let ordinal = (self.engine.ordinal + self.engine.compute_stamp_base) % encode::STAMP_PERIOD; + let g = &self.graph.compute; + g.user_stamp.w32(0, ids::COMPUTE_SEED)?; + g.fw_stamp.w32(0, 0)?; + // The next compute (ordinal + 1) counts as ordinal 1 again. + let period = encode::STAMP_PERIOD; + self.engine.compute_stamp_base = (period - self.engine.ordinal % period) % period; + self.engine.target = 0; + self.engine.tail_ct = 0; + dev_info!( + self.graph.dev.as_ref(), + "G15: compute stamp renumbered from the seed at compute {} (ordinal {:#x})\n", + self.engine.ordinal, + ordinal + ); + Ok(()) + } + fn publish_prefix_into(&mut self, job: &Arc, f: &mut Flight) -> Result { // The PB backing is mapped into the job's VM before anything of the prefix is changed. // Nothing is published when that fails, so the error must not be EIO (which reports a @@ -691,6 +816,10 @@ impl Firmware { self.map_pb(&job.vm) .map_err(|e| if e == EIO { EFAULT } else { e })?; let overlap = !self.engine.flights.is_empty(); + if !overlap { + self.renumber_render_stamps()?; + self.renumber_compute_stamps()?; + } let mut mixed = tuning::MIXED_PREFIX; let mut pf = Prefix { n: self.engine.ordinal, @@ -892,11 +1021,22 @@ impl Firmware { hz_stop = Some(1); break; } - for k in 0..n2 { - match command(&j2, k).ok_or(EINVAL)? { - Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?, - Command::Render(_) => return Err(EINVAL), + j2.bind()?; + let encoded = (|| { + for k in 0..n2 { + match command(&j2, k).ok_or(EINVAL)? { + Command::Compute(c) => { + self.prefix_compute(&j2, c, &mut pf, f, false)? + } + Command::Render(_) => return Err(EINVAL), + } } + Ok(()) + })(); + if let Err(error) = encoded { + // Not in the horizon yet and nothing of it reached the firmware. + j2.release_binding(); + return Err(error); } computes += n2; j2.progress.set_sent(n2); @@ -951,7 +1091,7 @@ impl Firmware { self.engine.head = pf.c_head; } if renders != 0 { - f.ta_target = encode::stamp_seed(ids::TA_SEED, pf.draw); + f.ta_target = encode::stamp_seed(ids::TA_SEED, pf.draw + self.engine.stamp_base); self.engine.ta_target = f.ta_target; f.fs_target = pf.last_fs; self.engine.fs_target = pf.last_fs; @@ -1050,7 +1190,7 @@ impl Firmware { Queue, self.graph.dev, "G15: prefix ctx={} computes {} (n={} head {}) renders {} (draw {} ta/fs head {}/{}) behind {}: {:?}\n", - job.vm.slot, + job.slot(), computes, pf.n, pf.c_head, @@ -1087,8 +1227,8 @@ impl Firmware { later_render: bool, ) -> Result { let n = pf.n + 1; - let t = encode::stamp(n); - let ctx = job.vm.slot; + let t = encode::stamp(n + self.engine.compute_stamp_base); + let ctx = job.slot(); let ts = &cmd.timestamps; let mut lap = Lap::start(); // Nothing published yet on failure. @@ -1131,6 +1271,7 @@ impl Firmware { ts, ctx, n, + t, tuning::COMPUTE_FAULT_MASK, image(w)?, image(q)?, @@ -1184,11 +1325,7 @@ impl Firmware { } let j = pf.computes as usize; f.cstamp[j] = t; - f.cwait[j] = if pf.prev == Prev::Render { - pf.last_fs - } else { - 0 - }; + f.cwait[j] = (pf.prev == Prev::Render).then_some(pf.last_fs); g.ring.w64(8 * pf.c_head as usize, s.work.va)?; pf.c_head = (pf.c_head + 1) % sizes::QUEUE_CAP; lap.lap(&mut self.engine.prof.write_ns); @@ -1233,9 +1370,10 @@ impl Firmware { let k = f.claims.pass[pf.renders as usize] as usize; let n = pf.draw + 1; let usc = job.queue.usc_base; - let ctx = job.vm.slot; - let tta = encode::stamp_seed(ids::TA_SEED, n); - let tfs = encode::stamp_seed(ids::FS_SEED, n); + let ctx = job.slot(); + let sn = n + self.engine.stamp_base; + let tta = encode::stamp_seed(ids::TA_SEED, sn); + let tfs = encode::stamp_seed(ids::FS_SEED, sn); // Early tiling: a later render member after a render of the same prefix whose render // dependencies are fragment-only (FRAGMENT_BARRIERS) and that has no start timestamps // lets its TA wait for the previous TA; its FS still waits for the previous FS, then @@ -1334,7 +1472,7 @@ impl Firmware { } else if early { let b = encode::barrier( r.fw_stamp[0].va, - encode::stamp_seed(ids::TA_SEED, n - 1), + encode::stamp_seed(ids::TA_SEED, sn - 1), 0, tta, RENDER_WAIT_UUID, @@ -1354,10 +1492,9 @@ impl Firmware { } else { let mut prev_fs = if pf.prev != Prev::None { pf.last_fs - } else if n == 1 { - ids::FS_SEED } else { - encode::stamp_seed(ids::FS_SEED, n - 1) + // With no offset, ordinal 0 is the FS seed itself (the first render). + encode::stamp_seed(ids::FS_SEED, sn - 1) }; // Restricted type 0: predecessor render in this prefix, no InitBM/resync. let internal = pf.prev == Prev::Render diff --git a/drivers/gpu/drm/asahi/g15/gate.rs b/drivers/gpu/drm/asahi/g15/gate.rs index f41b2e3f75f57..590f5e6f0a5c1 100644 --- a/drivers/gpu/drm/asahi/g15/gate.rs +++ b/drivers/gpu/drm/asahi/g15/gate.rs @@ -3,11 +3,13 @@ //! Stream admission gates (safety nets for userspace encoders). //! //! VDM: the root stream's first word is 0x21000960. CDM walk from `cdm_ctrl_stream_base` through -//! links until TERMINATE (0x40000000), at most 8192 records and 64 links (a longer or cyclic -//! stream is rejected): barriers 0x60000160, 0x60001560, 0x600215fe, 0x600201fe; +//! links until TERMINATE (0x40000000), at most 64 links and [`CDM_MAX_BYTES`] of records (a +//! longer or cyclic stream is rejected): barriers 0x60000160, 0x60001560, 0x600215fe, 0x600201fe; //! link 0x2000XXXX; direct launch (40 B, the G15 layout), indirect launches 36 B //! (`w0 & 0xff7fffff == 0x08080000`) and 24 B (0x10080000); every program VA inside the queue's -//! 4 GiB USC window, 64-aligned, first 64 bytes readable. Client memory is read through the VM's +//! 4 GiB USC window, 64-aligned, first 64 bytes readable; an indirect launch's 12 argument bytes +//! readable. A program or argument page found good is not checked again in the same walk, and a +//! walk checks at most [`CDM_MAX_PAGE_CHECKS`] pages. Client memory is read through the VM's //! page tables with a point-of-coherency clean+invalidate of the lines ([`mmu::Vm::read_user`]: //! the GPU may have written it). `g15_stream_gate=0` disables both gates (encoder bring-up only). @@ -24,8 +26,19 @@ const CDM_LINK: u32 = 0x20000000; const CDM_DIRECT: u32 = 0x00080000; const CDM_INDIRECT_SIZES: u32 = 0x08080000; const CDM_INDIRECT: u32 = 0x10080000; -const CDM_MAX_RECORDS: u32 = 8192; const CDM_MAX_LINKS: u32 = 64; +/// The stream block the encoder links to when one fills (Mesa's 64 KiB control-stream blocks). +const CDM_BLOCK_BYTES: u64 = 0x10000; +/// Stream bytes walked at most: the root block and one block per admitted link. A stream of +/// that size holds every record the encoder can write before the link budget (a record count +/// does not: the 1024 tessellated draws of one multi-draw write more than 8192). +const CDM_MAX_BYTES: u64 = (CDM_MAX_LINKS as u64 + 1) * CDM_BLOCK_BYTES; +/// Page checks (a program's or an indirect launch's arguments, each under the VM lock) a walk +/// makes at most. The last page found good is not checked again, but the encoder's indirect +/// arguments move to a new page every few launches: a stream at the link budget makes about +/// 8100. Twice that leaves room; without a bound, a stream of distinct pages would make one per +/// 24-byte launch, about 177000 in the byte budget. +const CDM_MAX_PAGE_CHECKS: u32 = 16384; /// Rejections logged per boot. const MAX_REPORTS: u32 = 32; @@ -58,7 +71,7 @@ struct Window { impl Window { /// The `count` words at `va`, reading whole spans of up to 256 bytes (not past the page /// unless the record crosses it). Unreadable memory: EFAULT. - fn words(&mut self, vm: &mmu::Vm, va: u64, count: usize) -> Result<[u32; 10]> { + fn words(&mut self, vm: &mut mmu::VmReadGuard, va: u64, count: usize) -> Result<[u32; 10]> { let n = count * 4; let cached = self.len != 0 && va >= self.va @@ -86,24 +99,57 @@ impl Window { } } +/// The pages a walk has found good and the page checks it has made. +struct Checked { + program: Option, + args: Option, + count: u32, +} + +/// One page check of `[start, end)` under the walk's page budget: ENOSPC once the budget is +/// spent. +fn check(vm: &mut mmu::VmReadGuard, start: u64, end: u64, count: &mut u32) -> Result { + if *count >= CDM_MAX_PAGE_CHECKS { + return Err(ENOSPC); + } + *count += 1; + cover(vm, start, end, true, false) +} + /// A launch's program: inside the queue's 4 GiB USC window, 64-aligned, first block readable. -/// `checked` caches the last proven program page. -fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Option) -> Result { +/// The last proven program page is not checked again. +fn program(vm: &mut mmu::VmReadGuard, usc: u64, address: u64, checked: &mut Checked) -> Result { if address < usc || address - usc >= USC_WINDOW || address & 63 != 0 { return Err(EINVAL); } // 64-aligned: the block never crosses its page. let page = address & !(mmu::UAT_PGMSK as u64); - if *checked == Some(page) { + if checked.program == Some(page) { return Ok(()); } - cover(vm, address, address + 64, true, false)?; - *checked = Some(page); + check(vm, address, address + 64, &mut checked.count)?; + checked.program = Some(page); + Ok(()) +} + +/// An indirect launch's 12 argument bytes: readable. The last proven argument page is not +/// checked again; arguments that cross a page are checked each time. +fn arguments(vm: &mut mmu::VmReadGuard, address: u64, checked: &mut Checked) -> Result { + let end = address.checked_add(12).ok_or(EINVAL)?; + let page = address & !(mmu::UAT_PGMSK as u64); + let within = (end - 1) & !(mmu::UAT_PGMSK as u64) == page; + if within && checked.args == Some(page) { + return Ok(()); + } + check(vm, address, end, &mut checked.count)?; + if within { + checked.args = Some(page); + } Ok(()) } /// VDM admission: a normal root stream opens with exactly the G15 barrier. -pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result { +pub(crate) fn vdm(vm: &mut mmu::VmReadGuard, va: u64) -> Result { if !params::stream_gate() { return Ok(()); } @@ -119,7 +165,7 @@ pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result { /// CDM admission: walks the stream from its base through no-return links until TERMINATE, /// admitting only the known G15S records: direct launch (40), indirect launch (36/24), barriers /// (4), link (8). -pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { +pub(crate) fn cdm(vm: &mut mmu::VmReadGuard, usc: u64, base: u64) -> Result { if !params::stream_gate() { return Ok(()); } @@ -128,15 +174,19 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { len: 0, bytes: [0; 256], }; - let mut checked = None; + let mut checked = Checked { + program: None, + args: None, + count: 0, + }; let mut va = base; let mut links = 0; - let mut records = 0; - while records < CDM_MAX_RECORDS { - records += 1; + let mut walked = 0; + while walked < CDM_MAX_BYTES { let w = window.words(vm, va, 1)?; if CDM_BARRIERS.contains(&w[0]) { va += 4; + walked += 4; continue; } if w[0] == CDM_TERMINATE { @@ -152,6 +202,7 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { if va & 3 != 0 { return Err(reject("CDM link", va, &w)); } + walked += 8; continue; } if w[0] == CDM_DIRECT { @@ -162,9 +213,13 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { } match program(vm, usc, address, &mut checked) { Err(error) if error == EINVAL => return Err(reject("CDM program", va, &w)), + Err(error) if error == ENOSPC => { + return Err(reject("CDM page check budget", va, &w)) + } result => result?, } va += 40; + walked += 40; continue; } let kind = w[0] & 0xff7fffff; @@ -182,15 +237,20 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { return Err(reject("CDM indirect launch", va, &w)); } let result = program(vm, usc, address, &mut checked) - .and_then(|()| cover(vm, indirect, indirect + 12, true, false)); + .and_then(|()| arguments(vm, indirect, &mut checked)); match result { Err(error) if error == EINVAL => return Err(reject("CDM indirect", va, &w)), + Err(error) if error == ENOSPC => { + return Err(reject("CDM page check budget", va, &w)) + } result => result?, } - va += if sizes { 36 } else { 24 }; + let size = if sizes { 36 } else { 24 }; + va += size; + walked += size; continue; } return Err(reject("CDM", va, &w)); } - Err(reject("CDM record budget", va, &[])) + Err(reject("CDM byte budget", va, &[])) } diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index b949a4b9b61ed..095b9cb96d51f 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -26,8 +26,10 @@ bool asahi_g15_board(void); bool asahi_g15_machine(const char *compatible); void asahi_g15_hold_power(struct device *dev); void asahi_g15_release_power(struct device *dev); -void asahi_g15_module_get(void); +bool asahi_g15_module_try_get(void); +void asahi_g15_module_put(void); void asahi_g15_reprobe(void); +void asahi_g15_report_loader_gpu(void); int asahi_g15_adt_phram(u64 *start, u64 *size); bool asahi_g15_ram_disjoint(u64 start, u64 size); bool asahi_g15_ram_within(u64 start, u64 size); @@ -35,10 +37,53 @@ unsigned int asahi_g15_memory_mode(u64 start, u64 size); void __iomem *asahi_g15_ioremap_wc(u64 start, size_t size); bool asahi_g15_dma_coherent(struct device *dev); int asahi_g15_pmp_prepare(struct device *dev); +int asahi_g15_display_prepare(struct device *dev); int asahi_g15_pmp_get(struct device *dev); +int asahi_g15_pmp_ready(void); void asahi_g15_pmp_put(struct device *dev); bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size); +/* + * The boot loader enables the T6030/T6031 GPU node only when its + * chosen.asahi,t603x-gpu setting is on; otherwise the node stays disabled, + * the adapter never probes and nothing says why native G15 is absent. Name + * the setting once, at module activation, on a machine whose /chosen has it + * and whose GPU node is disabled. Nothing else changes. + */ +void asahi_g15_report_loader_gpu(void) +{ + static const struct { + const char *prop; + const char *compatible; + } socs[] = { + { "asahi,t6031-gpu", "apple,agx-t6031" }, + { "asahi,t6030-gpu", "apple,agx-t6030" }, + }; + struct device_node *chosen, *np; + const char *value; + unsigned int i; + u32 cell; + + chosen = of_find_node_by_path("/chosen"); + if (!chosen) + return; + for (i = 0; i < ARRAY_SIZE(socs); i++) { + if (!of_property_present(chosen, socs[i].prop)) + continue; + np = of_find_compatible_node(NULL, NULL, socs[i].compatible); + if (np && !of_device_is_available(np)) { + if (!of_property_read_string(chosen, socs[i].prop, &value)) + pr_info("asahi: G15: native G15 not started: the boot loader left the GPU node %pOF disabled (/chosen/%s = \"%s\")\n", + np, socs[i].prop, value); + else if (!of_property_read_u32(chosen, socs[i].prop, &cell)) + pr_info("asahi: G15: native G15 not started: the boot loader left the GPU node %pOF disabled (/chosen/%s = %u)\n", + np, socs[i].prop, cell); + } + of_node_put(np); + } + of_node_put(chosen); +} + /* * The maintained m1n1 handoff keeps these four UAT regions in the linear map. * Require the exact enabled reserved-memory owner, in addition to the ADT @@ -46,20 +91,31 @@ bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size); */ bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size) { - static const char *const paths[] = { - "/reserved-memory/uat-ttbs", - "/reserved-memory/uat-pagetables", - "/reserved-memory/uat-handoff", - "/reserved-memory/uat-pagetables-l2", + static const char *const names[] = { + "uat-ttbs", + "uat-pagetables", + "uat-handoff", + "uat-pagetables-l2", }; - struct device_node *np; + struct device_node *rmem, *child, *np = NULL; struct resource res; + unsigned int count = 0; bool match; - if (index >= ARRAY_SIZE(paths)) + if (index >= ARRAY_SIZE(names)) return false; - np = of_find_node_by_path(paths[index]); - match = np && of_device_is_available(np) && + + /* Unit addresses belong to the DT layout, not the reservation name. */ + rmem = of_find_node_by_path("/reserved-memory"); + for_each_child_of_node(rmem, child) { + if (!of_node_name_eq(child, names[index])) + continue; + count++; + if (!np) + np = of_node_get(child); + } + of_node_put(rmem); + match = count == 1 && np && of_device_is_available(np) && !of_property_read_bool(np, "reusable") && !of_get_address(np, 1, NULL, NULL) && !of_address_to_resource(np, 0, &res) && @@ -90,6 +146,33 @@ int asahi_g15_pmp_prepare(struct device *dev) return apple_pmp_link_device(dev); } +/* + * T6031 has no PMP vote, but its first firmware start follows the native + * display as T6030's does: defer until that display is up. The display can + * be off for this boot (no handoff, or not requested), so say once why. An + * enabled display that is not published yet is said once too: if it never + * comes up (its DCP refused or failed to bind), that line is why the GPU + * stays off. + */ +int asahi_g15_display_prepare(struct device *dev) +{ + struct device_node *np; + int ret; + + if (!dev->of_node || !of_device_is_compatible(dev->of_node, "apple,agx-t6031")) + return -ENODEV; + ret = apple_drm_t6031_link_device(dev); + if (ret != -EPROBE_DEFER) + return ret; + np = of_find_compatible_node(NULL, NULL, "apple,t6031-display-subsystem"); + if (!np || !of_device_is_available(np)) + dev_info_once(dev, "G15: waiting for the T6031 native display, which this boot did not enable (it needs the boot loader's display handoff, and apple_t6031_display.enable=1 except on the J516C)\n"); + else + dev_info_once(dev, "G15: waiting for the T6031 native display to come up: the GPU firmware starts only after the display is published\n"); + of_node_put(np); + return ret; +} + int asahi_g15_pmp_get(struct device *dev) { if (!asahi_g15_board() || !dev->of_node || @@ -99,6 +182,11 @@ int asahi_g15_pmp_get(struct device *dev) } +int asahi_g15_pmp_ready(void) +{ + return apple_pmp_t6030_gpu_ready(); +} + /* * Give back the vote of asahi_g15_pmp_get() after a start that failed before * the firmware was touched. The full M3 stack's J514S bridge has no release: @@ -159,10 +247,20 @@ void asahi_g15_release_power(struct device *dev) } -/* Pin this module until reboot. */ -void asahi_g15_module_get(void) +/* + * Pin this module until reboot, unless its removal has begun: a deferred or + * asynchronous probe can run after an ordinary rmmod has passed its reference + * check, which __module_get() would not stop. + */ +bool asahi_g15_module_try_get(void) +{ + return try_module_get(THIS_MODULE); +} + +/* Drop a reference asahi_g15_module_try_get() took. */ +void asahi_g15_module_put(void) { - __module_get(THIS_MODULE); + module_put(THIS_MODULE); } /* diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index d566a54953e07..b211088310210 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -46,15 +46,12 @@ pub(crate) enum Platform { pub(crate) enum Board { /// The hash-checked T6030 templates ([`super::board`]). Templates, - /// The three images built from the device tree ([`super::board::record`], - /// [`super::board::device_tree`]) and cross-checked against a firmware file holding the - /// machine's own ([`super::board::blob`], pinned by its SHA-256), which also supplies the - /// record's words the device tree does not carry ([`super::board::record::FROM_FILE`]); a - /// difference refuses admission, before any firmware is touched. - Blob { - name: &'static kernel::str::CStr, - sha256: [u8; 32], - }, + /// The three images built from the device tree and the ADT's sensor rows + /// ([`super::board::record`], [`super::board::device_tree`]), with the SoC's reference + /// words for what neither carries ([`super::board::record::Reference`]). A value the + /// builder refuses stops admission, before any firmware is touched. + #[cfg(CONFIG_DRM_ASAHI_G15)] + DeviceTree(&'static super::board::record::Reference), /// No board data: admission refuses the SoC before anything firmware-visible. None, } @@ -163,6 +160,21 @@ pub(crate) mod t6030 { 0xe7, 0x9b, 0xb9, 0xb0, 0x22, 0x09, 0xdb, 0x53, ], }, + // The J514S (14-inch) boot chain loads the same B0 image: its + // measured TEXT digest, sizes and marker match the J516S row above. + FirmwareBuild { + name: "J514S B0 RTKit-2419.140.12 (maintained 14.x handoff)", + machine: Some(kernel::c_str!("apple,j514s")), + text_size: 0x5c000, + data_size: 0x114000, + marker: 0x58e88, + text_sha256: [ + 0xda, 0x26, 0xec, 0xca, 0xb1, 0xf8, 0x59, 0x8d, + 0xa8, 0x0c, 0x85, 0x5e, 0xca, 0x91, 0x65, 0x20, + 0x40, 0x6d, 0x1d, 0x06, 0x70, 0xf1, 0xd4, 0x70, + 0xe7, 0x9b, 0xb9, 0xb0, 0x22, 0x09, 0xdb, 0x53, + ], + }, ], machine: kernel::c_str!("apple,t6030"), platform: super::Platform::AdtWithPmp, @@ -516,8 +528,6 @@ pub(crate) mod sizes { pub(crate) const AHEAD_RECORDS: u32 = 64; /// Policy bound on logical queues reported in QUEUE_LIMITS (not a firmware limit). pub(crate) const MAX_QUEUES: u32 = 58; - /// TTBAT user slots: user VMs hold one each for their lifetime (ASIDs 1..63). - pub(crate) const MAX_VMS: u32 = 63; } /// Firmware channels, doorbells, seeds and identifiers. @@ -571,11 +581,25 @@ pub(crate) mod tuning { pub(crate) const SLOT_ROTATE: bool = true; /// Flights per VM. pub(crate) const MAX_INFLIGHT: u32 = 2; - /// PB growth through event 7 / DEVCTL 7, initial and maximum blocks. + /// PB growth through event 7 / DEVCTL 7, from the initial posting of [`PB_BLOCKS`]. pub(crate) const PB_GROW: bool = true; - /// Initial PB blocks, and the most that may be posted: one below the mapped heap's block - /// count, so the block-list producer (modulo K = 4096) never returns to 0. - pub(crate) const PB_BLOCKS: u32 = 1024; + /// PB blocks posted at (re)initialization: 64 blocks of 128 KiB (8 MiB). + pub(crate) const PB_BLOCKS: u32 = 64; + /// PB reclaim when the engine is idle: 0 never, 1 re-initialize, 2 also free the backing + /// above the initial posting and the reserve. + pub(crate) const PB_RECLAIM: u32 = 2; + /// Idle time before a PB reclaim, ms. + pub(crate) const PB_IDLE_MS: u32 = 1000; + /// Posted PB blocks above which an idle engine reclaims the PB. + pub(crate) const PB_RECLAIM_BLOCKS: u32 = 1024; + /// Renders before the TA stamp wrap within which the TA/FS stamps are renumbered while + /// nothing is in flight (the wrap is at about 16.7 million renders). + pub(crate) const STAMP_RESEED_MARGIN: u64 = 100_000; + /// Computes before the compute stamp wrap within which the compute stamp is renumbered + /// while nothing is in flight (the wrap is at 4128768 computes). + pub(crate) const COMPUTE_RESEED_MARGIN: u64 = 100_000; + /// The most PB blocks that may be posted: one below the mapped heap's block count, so the + /// block-list producer (modulo K = 4096) never returns to 0. pub(crate) const PB_MAX_BLOCKS: u32 = super::sizes::PB_BLOCKS - 1; /// At the host deadline, arm the firmware kill. pub(crate) const HANG_PREEMPT: bool = true; diff --git a/drivers/gpu/drm/asahi/g15/hw/t6031.rs b/drivers/gpu/drm/asahi/g15/hw/t6031.rs index 4af32f6068cd7..8bac33989db64 100644 --- a/drivers/gpu/drm/asahi/g15/hw/t6031.rs +++ b/drivers/gpu/drm/asahi/g15/hw/t6031.rs @@ -8,9 +8,8 @@ //! device tree node the boot loader fills, not an ADT export), the GPU power (its power domain; //! there is no PMP vote), the initdata root version, a longer HardwareData record (0xb480 //! bytes, its fourth view at +0xb400) placed aligned and ASC-cached, the IOMapping table (17 -//! slots at the T6031 register blocks) and the board images, built from the device tree and -//! checked against this machine's firmware file, which supplies the record's non-device-tree -//! words. Every +//! slots at the T6031 register blocks) and the board images, built from the device tree, the +//! ADT's sensor rows and the J516C reference words. Every //! value was measured on a J516C: the SGX identification //! registers, the loaded firmware segment, the device tree the boot loader hands over and the //! initdata of that machine. @@ -117,18 +116,6 @@ pub(crate) const LAYOUT: Layout = Layout { record_empty_writable: &RECORD_EMPTY_WRITABLE, }; -/// The J516C board file (`asahi/t6031-j516c-board.bin`, built into the kernel): the hardware -/// record, Globals and the power block that this machine's firmware accepted, as the earlier -/// J516C driver uploaded them (its device-tree-built records with the per-board words of this -/// machine applied). Kept out of the tree; pinned by its SHA-256. The three images are built -/// from the device tree and must match the file's; the file supplies the record's words the -/// device tree does not carry (`board::record::FROM_FILE`). -const BOARD_SHA256: [u8; 32] = [ - 0x4e, 0x03, 0xf2, 0x9f, 0xa4, 0x47, 0xdc, 0xeb, 0xfd, 0x4d, 0x1c, 0x22, 0x5a, 0x64, 0x7b, - 0xfa, 0x31, 0xb3, 0x63, 0x45, 0x8f, 0x28, 0x33, 0x82, 0x3a, 0x5c, 0xa8, 0xa1, 0x0e, 0xdf, - 0xf3, 0xfc, -]; - pub(crate) const CONFIG: Config = Config { name: "T6031 G15C", chip_id: 0x6031, @@ -172,15 +159,12 @@ pub(crate) const CONFIG: Config = Config { ], }, ], - // The pinned board images and measured layout belong to the J516C, not + // The reference words and measured layout belong to the J516C, not // every T6031 board (in particular, not the 14-inch J514C). machine: kernel::c_str!("apple,j516c"), platform: super::Platform::DeviceTree, layout: &LAYOUT, - board: super::Board::Blob { - name: kernel::c_str!("asahi/t6031-j516c-board.bin"), - sha256: BOARD_SHA256, - }, + board: super::Board::DeviceTree(&super::super::board::record::J516C), // Four clusters: the saved-state pointers at +0x2980 of the preemption buffer (0xd80 plus // 0x700 per cluster), UMA PTC 0x6000400020, DUPM 0x3c000000140 (32 N | 96 N << 32 for // N = 10 cores per cluster) and 0x1a0e9 = 0, with the saved-state pointers and the diff --git a/drivers/gpu/drm/asahi/g15/initdata.rs b/drivers/gpu/drm/asahi/g15/initdata.rs index f3f5232e92a28..e53cb44eff43a 100644 --- a/drivers/gpu/drm/asahi/g15/initdata.rs +++ b/drivers/gpu/drm/asahi/g15/initdata.rs @@ -160,61 +160,27 @@ fn templates( /// HardwareDataA word holding the system counter at initdata creation. const HWDATA_A_INIT_TIMESTAMP: usize = 0x424c; -/// Board images of a SoC with a firmware file. The hardware record ([`board::hardware_record`]), -/// Globals and Power ([`board::device_tree`]) are built from the device tree; the file, this -/// machine's own, supplies the record's words the device tree does not carry and is otherwise -/// a cross-check: admission ([`prepare_board`], before any firmware is touched) is refused at -/// the first differing word. Returns the images and the top operating-point frequency in kHz. -fn blob( +/// Board images of a device-tree SoC: the hardware record ([`board::hardware_record`]) from the +/// GPU node, the ADT's sensor rows and the SoC's `reference` words, Globals and Power +/// ([`board::device_tree`]) from the GPU node and that record. Returns the images and the top +/// operating-point frequency in kHz. +fn device_tree_board( dev: &device::Device, cfg: &Config, adt: &GpuAdt, - name: &CStr, - sha256: &[u8; 32], + reference: &board::record::Reference, ) -> Result<(KVVec, KVVec, KVVec, u32)> { - let layout = cfg.layout; - let images = board::blob::load( - dev, - name, - sha256, - [layout.hwdata_size, globals::SIZE, power::SIZE], - )?; - let hw_img = board::hardware_record(dev, cfg, adt, &images.hardware)?; + let hw_img = board::hardware_record(dev, cfg, adt, reference)?; let (glob, pwr, max_power_mw) = board::device_tree(dev, &hw_img)?; - for (what, built, file) in [ - ("hardware record", &hw_img, &images.hardware), - ("Globals", &glob, &images.globals), - ("Power", &pwr, &images.power), - ] { - let diff = if what == "hardware record" { - board::record::cross_check(built, file) - } else { - board::words::first_difference(built, file) - }; - if let Some((at, b, f)) = diff { - dev_err!( - dev, - "board data: device-tree {} differs from {} at +{:#x}: built {:#x}, file {:#x}\n", - what, - name, - at, - b, - f - ); - return Err(EINVAL); - } - } let max_mhz = adt.perf_mhz.iter().copied().max().unwrap_or(0); dev_info!( dev, - "board data: {} images from the device tree match {} ({} record words from the file, {} file words left zero), {} mW maximum power, {} MHz top operating point\n", + "board data: {} images from the device tree and the ADT sensor rows, reference maximum power {} mW and MTR masks, {} MHz top operating point\n", cfg.name, - name, - board::record::FROM_FILE.len(), - board::record::UNNEEDED.len(), max_power_mw, max_mhz ); + dev_info!(dev, "board data sha256 {}\n", Hex(&glue::sha256(&hw_img))); let max_khz = max_mhz.checked_mul(1000).ok_or_else(|| { dev_err!(dev, "board data: top operating point {} MHz does not fit in kHz\n", max_mhz); EINVAL @@ -225,8 +191,9 @@ fn blob( /// Verified board bytes, prepared before any GPU firmware is started. pub(crate) type BoardImages = (KVVec, KVVec, KVVec, u32); -/// Check availability, digest, layout and machine-derived tables before firmware start. -/// Keep the verified bytes so a later firmware request cannot change the admitted data. +/// Build and check the board images before any GPU firmware is started: the templates against +/// their digests and layout, or the device-tree record from the GPU node, the ADT and the SoC's +/// reference words. pub(crate) fn prepare_board( dev: &device::Device, cfg: &Config, @@ -234,7 +201,7 @@ pub(crate) fn prepare_board( ) -> Result { match cfg.board { hw::Board::Templates => templates(dev, adt), - hw::Board::Blob { name, sha256 } => blob(dev, cfg, adt, name, &sha256), + hw::Board::DeviceTree(reference) => device_tree_board(dev, cfg, adt, reference), hw::Board::None => Err(ENODEV), } } @@ -296,7 +263,7 @@ pub(crate) fn build( // Board images, verified, then the owned replacements. let (mut hw_img, mut glob, pwr, max_khz) = board_images; - if matches!(cfg.board, hw::Board::Blob { .. }) { + if matches!(cfg.board, hw::Board::DeviceTree(_)) { // This timestamp belongs to initdata creation, not board-data preflight. let now: u64; // SAFETY: Reading the architectural counter has no side effects. @@ -318,10 +285,10 @@ pub(crate) fn build( } put64(&mut hw_img, hwdata::TIMESTAMPS, va::TIMESTAMPS)?; put32(&mut glob, globals::UNK_980, 100)?; - // The device tree's standby time (a machine with a firmware file; the templates' machine + // The device tree's standby time (a device-tree SoC; the templates' machine // has none). let dt_standby = match cfg.board { - hw::Board::Blob { .. } => { + hw::Board::DeviceTree(_) => { let prop = board::words::PROPS[board::words::STANDBY_TIMER]; match dev .of_node() @@ -369,8 +336,8 @@ pub(crate) fn build( globals::IDLE_PREDICTIVE, hw::tuning::IDLE_PREDICTIVE as u32, )?; - // HardwareDataA at +0x4580: maintained S8 request (the templates' machine; a board image - // from a file carries its own machine's words). + // HardwareDataA at +0x4580: maintained S8 request (the templates' machine; a record built + // from the device tree carries its own state words). if cfg.board == hw::Board::Templates { for (offset, value) in [ (0x2c, 8), diff --git a/drivers/gpu/drm/asahi/g15/job.rs b/drivers/gpu/drm/asahi/g15/job.rs index 5f6669f06d1b0..a93b6039cc466 100644 --- a/drivers/gpu/drm/asahi/g15/job.rs +++ b/drivers/gpu/drm/asahi/g15/job.rs @@ -21,30 +21,36 @@ //! The engine owns an `Arc` from [`super::Firmware::enqueue`] on. It completes the job //! exactly once with [`Job::complete`] (later calls are ignored), through [`Completions`] //! collected under the device lock and [`complete_all`] after it is dropped. A job whose storage -//! the firmware may still reference stays in `Engine::retained` (failed jobs until reboot): its -//! [`mmu::VmJobGuard`] keeps the VM's mappings and its VM state keeps the VM's TTBAT slot until +//! the firmware may still reference after a fail-stop stays in `Engine::retained` until reboot: its +//! [`mmu::VmJobGuard`] keeps the VM's mappings and its binding keeps the VM's TTBAT slot until //! the last `Arc` drops. -use core::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use kernel::{ bindings, c_str, device, dma_fence::{Fence, RawDmaFence}, drm::sched, + new_mutex, prelude::*, - sync::Arc, + sync::{Arc, Mutex}, }; use super::{command::Command, fence::JobFence, hw, queue::QueueState, vm::VmState, Shared}; use crate::mmu; /// A job as the engine sees it. +#[pin_data] pub(crate) struct Job { /// Submission ID (debugging). pub(crate) id: u64, /// The submitting queue (priority, service turn, horizon history). pub(crate) queue: Arc, - /// The job's VM (TTBAT slot, status). + /// The job's VM and status. pub(crate) vm: Arc, + /// Hardware context, assigned immediately before the engine publishes the job. + context: AtomicU32, + #[pin] + binding: Mutex>, /// Validated commands, in submission order (at most 64). pub(crate) commands: KVec, /// The job's hardware fence. @@ -73,11 +79,13 @@ impl Job { guard: mmu::VmJobGuard, ) -> Result> { let progress = super::engine::Progress::new(commands.len()); - Ok(Arc::new( - Self { + Arc::pin_init( + pin_init!(Self { id, queue, vm, + context: AtomicU32::new(0), + binding <- new_mutex!(None, "g15::Job::binding"), commands, fence, progress, @@ -85,9 +93,29 @@ impl Job { _guard: guard, accepted: AtomicBool::new(false), started: AtomicBool::new(false), - }, + }), GFP_KERNEL, - )?) + ) + } + + pub(crate) fn slot(&self) -> u32 { + self.context.load(Ordering::Relaxed) + } + + /// Called under the device lock immediately before publication. + pub(crate) fn bind(&self) -> Result { + let mut slot = self.binding.lock(); + if slot.is_none() { + let binding = self.vm.try_bind()?; + self.context.store(binding.slot(), Ordering::Relaxed); + *slot = Some(binding); + } + Ok(()) + } + + /// All commands retired successfully, or nothing of the job was published. + pub(crate) fn release_binding(&self) { + drop(self.binding.lock().take()); } /// Completes the job: latches the VM status on failure, then signals the fence (after every @@ -120,6 +148,9 @@ impl Job { self.vm.status.record(error); } } + if result.is_ok() || !engine { + self.release_binding(); + } self.fence.signal(result); } diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 713a1360b5687..9ffd858598b1c 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -14,9 +14,9 @@ //! //! # Plugging into the shared driver //! -//! - [`Driver`] is a separate platform driver. T6030 has a module alias; its probe must wait -//! for the native display and PMP supplier before booting firmware. T6031 retains explicit -//! module loading. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. +//! - [`Driver`] is a separate platform driver with a module alias. Its probe must wait for +//! the native display (T6030 and T6031) and the PMP supplier (T6030) before booting +//! firmware. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. //! - [`Gpu`] implements [`crate::gpu::Gpu`]: GET_PARAMS identity, VM creation (`new_vm`), //! queues (`new_queue`, see [`queue`]) and timestamp buffers. //! - User VMs are shared-layer [`mmu::Vm`]s on a [`mmu::Uat::new_adopted`] UAT with tracked @@ -45,6 +45,7 @@ mod adt; mod board; mod boot; +mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; @@ -151,13 +152,15 @@ const OF_TABLE: kernel::device_id::IdArray = ), ]); -// T6030 probe admission requires the native display and PMP supplier before -// firmware starts. T6031 retains explicit loading until it has an equivalent -// display-order admission gate. +// Probe admission requires the native display before firmware starts (T6030 +// also its PMP supplier), so both may be loaded at device creation. kernel::of_device_table!( AUTOLOAD_OF_TABLE, @none, - [of::DeviceId::new(c_str!("apple,agx-t6030"))] + [ + of::DeviceId::new(c_str!("apple,agx-t6030")), + of::DeviceId::new(c_str!("apple,agx-t6031")), + ] ); /// The G15 platform adapter: the shared DRM registration probed through @@ -226,6 +229,8 @@ static MODULE_READY: AtomicBool = AtomicBool::new(false); /// No fallible module initialization may follow this call. pub(crate) fn activate() { MODULE_READY.store(true, Ordering::Release); + // SAFETY: The helper only reads the device tree. + unsafe { boot::glue::asahi_g15_report_loader_gpu() }; // SAFETY: Both pinned platform registrations are fully initialized and remain owned // by DriverModule. The C helper borrows their live driver pointer only for this call. unsafe { boot::glue::asahi_g15_reprobe() }; @@ -376,7 +381,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. + /// Live VMs and the client aliases of every user VM. pub(crate) vms: Arc, } @@ -454,17 +459,18 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain(); + gpu.retain()?; Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the + /// Takes the module reference once; `boot::start` already took it before touching the + /// firmware, so this only confirms it. Nothing firmware-reachable is freed on any path: the /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a /// failed start) until reboot. - pub(crate) fn retain(&self) { - boot::glue::retain_module(); + pub(crate) fn retain(&self) -> Result { + boot::glue::retain_module() } } @@ -539,7 +545,21 @@ impl Firmware { vms, }, ); - if let Err(e) = firmware.init_graphs() { + let result = firmware.init_graphs().and_then(|_| { + if cfg.platform != hw::Platform::AdtWithPmp { + return Ok(()); + } + // A report before firmware startup is ACKed without enabling bandwidth scaling. + // Reassert it after setup, including the backend's final device control. + // SAFETY: This session retains its acknowledged T6030 AGX power vote. + let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; + if ret == 0 { + Ok(()) + } else { + Err(Error::from_errno(ret)) + } + }); + if let Err(e) = result { let Firmware { session, init, diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index b1e3218fafc8d..c2c3486475061 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -18,6 +18,30 @@ use crate::module_parameters; /// updates); this bounds the kernel's staging allocation. pub(crate) const MAX_BINDS: u32 = 1 << 20; +// At least one prefix of renders, so that the last prefix published outside the margin cannot +// cross the wrap; at most half the stamp range. +static_assert!( + hw::tuning::STAMP_RESEED_MARGIN >= hw::tuning::PREFIX_RENDERS as u64 + && hw::tuning::STAMP_RESEED_MARGIN <= super::encode::RENDER_WRAP / 2 +); +// At least one prefix of computes; at most half the stamp range. +static_assert!( + hw::tuning::COMPUTE_RESEED_MARGIN >= hw::sizes::COMPUTE_SLOTS as u64 + && hw::tuning::COMPUTE_RESEED_MARGIN <= super::encode::COMPUTE_WRAP / 2 +); + +/// Renders before the TA stamp wrap within which the TA/FS stamps are renumbered from their seeds, +/// at the first moment with nothing in flight ([`hw::tuning::STAMP_RESEED_MARGIN`]). +pub(crate) fn stamp_reseed_margin() -> u64 { + hw::tuning::STAMP_RESEED_MARGIN +} + +/// Computes before the compute stamp wrap within which the compute stamp is renumbered from its +/// seed, at the first moment with nothing in flight ([`hw::tuning::COMPUTE_RESEED_MARGIN`]). +pub(crate) fn compute_reseed_margin() -> u64 { + hw::tuning::COMPUTE_RESEED_MARGIN +} + /// Host deadline of a published prefix without progress, ms (at least 100). pub(crate) fn deadline_ms() -> u32 { module_parameters::g15_deadline_ms.value().max(100) @@ -64,13 +88,33 @@ pub(crate) fn accept_text_prefix() -> u64 { } /// The parameter for the PB growth maximum the firmware is given, in blocks, clamped to -/// 64..=4095. [`super::pools::Pools::soft_max`] raises it to at least the initial 1024 blocks; -/// the blocks above it, up to the 4095 posted at most, answer the synchronous requests of -/// partial renders. It is read-only after boot and does not change the 512 MiB heap. +/// 64..=4095. [`super::pools::Pools::soft_max`] raises it to at least the initial posting; the +/// blocks above it, up to the 4095 posted at most, answer the synchronous requests of partial +/// renders. It is read-only after boot and does not change the heap's backing. pub(crate) fn pb_max_blocks() -> u32 { module_parameters::g15_pb_max_blocks.value().clamp(64, super::hw::tuning::PB_MAX_BLOCKS) } +/// The initial PB posting, in blocks: a fixed driver value, not a module parameter. +pub(crate) fn pb_blocks() -> u32 { + super::hw::tuning::PB_BLOCKS +} + +/// PB reclaim at idle: 0 never, 1 re-initialize, 2 re-initialize and free the backing. +pub(crate) fn pb_reclaim() -> u32 { + super::hw::tuning::PB_RECLAIM +} + +/// Idle time before a PB reclaim, ns. +pub(crate) fn pb_idle_ns() -> u64 { + super::hw::tuning::PB_IDLE_MS as u64 * 1_000_000 +} + +/// Posted blocks above which an idle engine reclaims the PB. +pub(crate) fn pb_reclaim_blocks() -> u32 { + super::hw::tuning::PB_RECLAIM_BLOCKS +} + /// Whether FRAGMENT_BARRIERS is advertised and mapped to early tiling. pub(crate) fn early_tiling() -> bool { module_parameters::g15_early_tiling.value() != 0 diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 566995eb342c9..b0356376ecf96 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -23,7 +23,7 @@ use kernel::{ hrtimer::{ HrTimerCallback, HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode, }, - Delta, Jiffies, Monotonic, + msecs_to_jiffies, Delta, Jiffies, Monotonic, }, workqueue::RawWorkItem, }; @@ -85,7 +85,15 @@ impl Shared { let evented = self.evented.swap(false, Ordering::AcqRel); match firmware.poll(evented, &mut done) { Ok(true) => self.rearm(firmware.engine.chan_wait), - Ok(false) => {} + // Idle: nothing polls until new work arrives, so a PB reclaim that comes + // due with more idle time gets one more pass then. A job's arrival brings + // the poll work forward (`rearm`), which replaces this expiry. + Ok(false) => { + if let Some(ns) = firmware.reclaim_delay_ns() { + let ms = ns.div_ceil(1_000_000).min(u32::MAX as u64) as u32; + self.mod_poll(msecs_to_jiffies(ms) + 1); + } + } Err(error) => self.fail_stop(error), } } @@ -109,7 +117,7 @@ impl Shared { if let Err(error) = firmware.publish(&mut done) { self.fail_stop(error); } - if !firmware.engine.flights.is_empty() { + if !firmware.engine.flights.is_empty() || firmware.engine.slot_wait { self.rearm(firmware.engine.chan_wait); } } @@ -181,10 +189,7 @@ impl Shared { } } -/// Stamp cell `obs` reached its own target, bounded by the newest published target `tail`. -pub(crate) fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { - obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) -} +pub(crate) use super::recovery::logic::stamp_reached; /// Cursor `c` of a ring of `m` entries passed end `h` of entries from `b`, within tail `t`. pub(crate) fn ring_passed(c: u32, b: u32, h: u32, t: u32, m: u32) -> bool { @@ -207,7 +212,10 @@ impl Firmware { // prefixes published the retirement loop below drains first. if self.engine.flights.is_empty() { self.drain_rings()?; - if self.recovery_idle() { + // Idle bookkeeping of the PB: when the idle began, and the reclaim once it is due. + self.note_idle(); + self.reclaim_check(); + if self.recovery_idle() && self.engine.pending.is_empty() { return Ok(false); } } @@ -228,8 +236,14 @@ impl Firmware { self.engine.prof.polls += 1; self.engine.prof.poll_ns += now_ns().wrapping_sub(start); } + // The pass that retired the last flight is not re-armed: note the idle here, not only + // at the start of a pass that finds the flights empty. + self.note_idle(); published?; - Ok(!self.engine.flights.is_empty() || self.recovery.needs_repoll() || self.engine.frozen) + Ok(!self.engine.flights.is_empty() + || self.engine.slot_wait + || self.recovery.needs_repoll() + || self.engine.frozen) } /// Drains the rings and retires finished flights, with up to four trailing 5 us polls @@ -448,6 +462,10 @@ impl Firmware { p.count.store(p.sent(), Ordering::Relaxed); } } + if result.is_ok() && p.next() == p.sent() { + // All published prefixes retired; another VM may run before this job resumes. + job.release_binding(); + } if p.next() < p.sent() || p.sent() < p.count() { return; // more of it published or still to publish } @@ -465,6 +483,7 @@ impl Firmware { match result { Err(error) => complete(done, j2, Err(error)), Ok(()) => { + j2.release_binding(); s.jobs += 1; s.job_cmds += p2.count() as u64; s.job_max = s.job_max.max(p2.count() as u64); @@ -484,23 +503,25 @@ impl Firmware { return; }; let (g, r, e) = (&self.graph.compute, &self.graph.render, &self.engine); - let (mut c, mut ta, mut fs) = (0, 0, 0); + let pending = super::recovery::logic::pending_stamp; + let (mut c, mut ta, mut fs) = (None, None, None); if f.computes != 0 { - c = g.fw_stamp.r32(0).unwrap_or(0); - if stamp_reached(c, f.target, e.target) { - c = 0; // its computes are done: later work does not count - } + c = g + .fw_stamp + .r32(0) + .ok() + .and_then(|v| pending(v, f.target, e.target)); } if f.renders != 0 { - ta = r.fw_stamp[0].r32(0).unwrap_or(0); - fs = r.fw_stamp[1].r32(0).unwrap_or(0); - if stamp_reached(ta, f.ta_target, e.ta_target) { - ta = 0; - } - if stamp_reached(fs, f.fs_target, e.fs_target) { - fs = 0; - } + ta = r.fw_stamp[0] + .r32(0) + .ok() + .and_then(|v| pending(v, f.ta_target, e.ta_target)); + fs = r.fw_stamp[1] + .r32(0) + .ok() + .and_then(|v| pending(v, f.fs_target, e.fs_target)); } let moved = e.progress_flight == f.published_ns && [c, ta, fs] != e.progress; let published = f.published_ns; @@ -606,7 +627,7 @@ impl Firmware { self.graph.dev.as_ref(), "G15: {}: ctx={} prefixes={} n={} target={:#x} user={:#x} fw={:#x} done={} read={} write={} head={} ch={}/{} events={}/{} errors={} busy={} fault={:#x} {}ms\n", what, - f.map_or(0, |f| f.job.vm.slot), + f.map_or(0, |f| f.job.slot()), e.flights.len(), e.ordinal, e.target, @@ -716,6 +737,16 @@ impl Firmware { s.hz_ahead )?; self.pools.write_stats(f)?; + // What is left before the stamps wrap, and the renumbering margins. + let (compute_left, render_left) = self.stamps_left(); + writeln!( + f, + "stamps: compute_left {} compute_margin {} render_left {} render_margin {}", + compute_left, + params::compute_reseed_margin(), + render_left, + params::stamp_reseed_margin() + )?; writeln!( f, "stage: ta_us {} fs_us {} cdm_us {} span_us {}", diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index c635ed11c4a0c..668d7b408dcc4 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -6,16 +6,30 @@ //! (host-tested). //! //! - One 512 MiB PB heap at [`super::hw::client::PB_HEAP`] (4096 blocks of 128 KiB), backed on -//! demand: the initial blocks when the GPU starts, then chunks of at most -//! [`growth::CHUNK_BLOCKS`] (4 MiB) as growth needs them ([`Firmware::back_pb`]), never freed -//! or shrunk. The backing ([`PbChunk`]) is mapped only into user VMs, at the fixed client GVA, -//! each VM's before its prefixes are published ([`Firmware::map_pb`]); nothing names its pages -//! through the host window or context 0 (the records hold client GVAs only). Initial -//! [`super::hw::tuning::PB_BLOCKS`] (1024), at most [`super::hw::tuning::PB_MAX_BLOCKS`] -//! (4095) posted, so the producer never returns to 0; the firmware is given -//! [`Pools::soft_max`] (default 3072) as its growth maximum. Truthful K = 4096 in every +//! demand in chunks of [`growth::CHUNK_BLOCKS`] blocks (16 MiB) from the heap base: when the +//! GPU starts, the chunks of the initial posting and one reserve chunk +//! ([`growth::Ledger::chunks_for`]). The backing ([`PbChunk`]) is mapped only into user VMs, at +//! the fixed client GVA: each VM's before its prefixes are published ([`Firmware::map_pb`]), +//! and a chunk backed later also into the VM of the flights; nothing names its pages through +//! the host window or context 0 (the records hold client GVAs only). Initial +//! `tuning::PB_BLOCKS` (64), at most [`super::hw::tuning::PB_MAX_BLOCKS`] (4095) posted, so the +//! producer never returns to 0; the firmware is given [`Pools::soft_max`] (default 3072) as +//! its growth maximum. Truthful K = 4096 in every //! profile (PB reconstruction, also in recovery, synchronizes 8 * K bytes of the block list: //! K = 0x26cd overran the 0x8000-byte list and crashed the second recovery). +//! - Backing: a request is granted from the backed blocks above the posted count; after the +//! replies of a pass, while fewer than a reserve chunk remain, [`Firmware::refill_backing`] +//! backs the next chunk, so a reply never waits for the allocator. Only a reserve that ran dry +//! makes a request wait: it stays queued, the pass backs a chunk and the next pass answers it. +//! - Reclaim ([`Firmware::reclaim_check`], `tuning::PB_RECLAIM`): posted blocks never come back +//! while a PB lives (the firmware recycles pages inside its own pool; the host has no view of +//! it), so an engine idle for `tuning::PB_IDLE_MS` with more than `tuning::PB_RECLAIM_BLOCKS` posted +//! re-initializes the PB at its next render through the re-initialization path of a recovered +//! abort (fresh lists and manager, InitBM of the initial blocks, no halt) and frees the backing +//! above the kept chunks (also with fewer posted, when more is backed than they need); an +//! engine idle for 50 ms (a client boundary) with more than half the maximum posted +//! re-initializes it the same way but keeps the backing; if no render follows, with +//! `tuning::PB_RECLAIM` 2 the backing above the kept chunks is freed after `tuning::PB_IDLE_MS`. //! - Growth: the event drain queues type-7 requests; [`Firmware::answer_growth`] answers each //! one in the poll pass: type 7, PB slot 0, the context of a live published render prefix, //! counter >= next expected, epoch == current halt count; write records `(G - 0x1000000000) @@ -41,7 +55,6 @@ mod growth; use core::fmt; -use core::sync::atomic::Ordering; use kernel::{ device, prelude::*, @@ -58,7 +71,7 @@ use super::Firmware; use crate::driver::AsahiDrmDevice as AsahiDevice; use crate::{gem, mmu}; -pub(crate) use growth::Ledger; +pub(crate) use growth::{Answer, IdleClock, Ledger}; static_assert!(buffer::MAX_BLOCKS == sizes::PB_BLOCKS); static_assert!(buffer::MAX_PAGES == sizes::PB_PAGES); @@ -68,54 +81,43 @@ static_assert!(buffer::MAX_BLOCKS as u64 * buffer::BLOCK_BYTES == client::PB_HEA static_assert!(buffer::PAGE_LIST_SIZE as u64 == client::PB_PAGE_LIST_SIZE); static_assert!(uma_manager::SEEDS == sizes::FREELIST_SEED); static_assert!(uma_pools::POOL_SIZE == sizes::POOL_SIZE); -// The ledger keeps the driver's PB sizes: neither is capped by [`growth::MAX_POSTED`]. +// The ledger keeps the driver's PB maximum: it is not capped by [`growth::MAX_POSTED`]. The +// initial posting (`tuning::PB_BLOCKS`) is clamped to MIN_BLOCKS..=PB_MAX_BLOCKS. static_assert!(tuning::PB_MAX_BLOCKS <= growth::MAX_POSTED); -static_assert!(buffer::MIN_BLOCKS <= tuning::PB_BLOCKS); -static_assert!(tuning::PB_BLOCKS <= tuning::PB_MAX_BLOCKS); +static_assert!(buffer::MIN_BLOCKS <= tuning::PB_MAX_BLOCKS); // The PB alias is GPU-only (not cached and firmware-visible): dropping one is a plain unmap, // with no firmware cache flush (`KernelMapping::drop`). static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent()); +static_assert!(growth::MAX_CHUNKS * growth::CHUNK_BLOCKS == buffer::MAX_BLOCKS); -/// Physical backing of PB blocks `[first, first + blocks)`: a zeroed GEM kernel object, pinned -/// and DMA-mapped ([`object::gpu_backing`]), aliased only into user VMs at the blocks' fixed -/// client GVA ([`Firmware::map_pb`]). Retained until reboot, like every firmware object; each -/// alias also holds the object. +/// Bytes of one backing chunk. +pub(crate) const CHUNK_BYTES: usize = growth::CHUNK_BLOCKS as usize * buffer::BLOCK_BYTES as usize; + +/// Client GVA of backing chunk `i`. +pub(crate) const fn chunk_gva(i: usize) -> u64 { + client::PB_HEAP + (i * CHUNK_BYTES) as u64 +} + +/// Physical backing of one chunk of PB blocks: a zeroed GEM kernel object of [`CHUNK_BYTES`], +/// pinned and DMA-mapped ([`object::gpu_backing`]), aliased only into user VMs at its fixed +/// client GVA ([`chunk_gva`], [`VmState::map_heap`]). Each alias also holds the object. struct PbChunk { - first: u32, - blocks: u32, _pages: HeldPages, backing: gem::ObjectRef, } impl PbChunk { - fn new(dev: &AsahiDevice, first: u32, blocks: u32) -> Result { - if blocks == 0 || first.checked_add(blocks).ok_or(EINVAL)? > buffer::MAX_BLOCKS { - return Err(EINVAL); - } - let (backing, pages) = - object::gpu_backing(dev, blocks as usize * buffer::BLOCK_BYTES as usize)?; + fn new(dev: &AsahiDevice) -> Result { + let (backing, pages) = object::gpu_backing(dev, CHUNK_BYTES)?; Ok(Self { - first, - blocks, _pages: pages, backing, }) } - /// The block after the chunk's last. - fn end(&self) -> u32 { - self.first + self.blocks - } - - /// Maps the whole chunk, exactly its object's size, at its fixed client GVA in `vm`. - fn map(&self, vm: &mmu::Vm) -> Result { - vm.map_at( - client::PB_HEAP + self.first as u64 * buffer::BLOCK_BYTES, - self.blocks as usize * buffer::BLOCK_BYTES as usize, - &self.backing.gem, - mmu::Prot::from_pte(pte::LEAF_CACHED), - false, - ) + /// The GEM object, for the VM mappings. + fn gem(&self) -> ARef { + self.backing.gem.clone() } } @@ -161,6 +163,13 @@ pub(crate) struct PbStats { pub(crate) grown: u64, pub(crate) refused: u64, pub(crate) bad: u64, + /// Requests that waited a pass for backing. + pub(crate) waits: u64, + /// Chunks backed after the GPU started. + pub(crate) refills: u64, + /// Idle re-initializations, and the chunks they freed. + pub(crate) reclaims: u64, + pub(crate) freed: u64, } /// Render allocator and PB state: the allocators, the PB ledger and the spare set. @@ -179,9 +188,8 @@ pub(crate) struct Pools { spare_ready: bool, pb_pages: Arc, pb_blocks: Arc, - /// The PB backing, contiguous from block 0, in block order: the initial blocks, then the - /// growth chunks. Its capacity, reserved up front, is the most the ledger can need - /// ([`growth::max_chunks`]). + /// The PB backing chunks, from the heap base (chunk `i` at [`chunk_gva`]`(i)`). Its + /// capacity, reserved up front, is the whole heap's [`growth::MAX_CHUNKS`]. chunks: KVec, /// The PB growth ledger since the last (re)initialization. pub(crate) ledger: Ledger, @@ -189,6 +197,11 @@ pub(crate) struct Pools { c1_written: bool, /// The unanswered-at-the-maximum warning was given since the last initialization. exhausted_logged: bool, + /// The engine's idle clock (nothing published, nothing pending), for the reclaim. + idle: IdleClock, + /// The pending re-initialization was set by a reclaim ([`Firmware::reclaim_check`]), not by + /// a recovery; cleared by [`Self::init`]. + reclaim_pending: bool, pub(crate) stats: PbStats, } @@ -376,11 +389,24 @@ impl Pools { 0, true, )?; - // Backing for the initial blocks; growth adds the rest. - let ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); - let mut chunks = - KVec::with_capacity(growth::max_chunks(ledger.initial, ledger.max), GFP_KERNEL)?; - chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?; + // Backing for the initial blocks and the reserve; growth adds the rest. + let mut ledger = Ledger::new( + tuning::PB_GROW, + super::params::pb_blocks(), + tuning::PB_MAX_BLOCKS, + ); + let mut chunks = KVec::with_capacity(growth::MAX_CHUNKS as usize, GFP_KERNEL)?; + for _ in 0..Ledger::chunks_for(ledger.initial) { + chunks.push(PbChunk::new(dev)?, GFP_KERNEL)?; + ledger.back_chunk(); + } + dev_info!( + dev.as_ref(), + "G15: PB heap: {} blocks posted at start, {} chunks of {} MiB backed\n", + ledger.initial, + chunks.len(), + CHUNK_BYTES >> 20 + ); Ok(Self { dev: dev.as_ref().as_ref().into(), drm: dev.into(), @@ -394,6 +420,8 @@ impl Pools { ledger, c1_written: false, exhausted_logged: false, + idle: IdleClock::default(), + reclaim_pending: false, stats: PbStats::default(), }) } @@ -451,9 +479,11 @@ impl Pools { // PB: 16384 pages of 0x8000 in 4096 blocks of 0x20000; with growth the first initial // blocks of them, at most 4095 posted, and page limits at the firmware's growth maximum. - // In every profile the block list declares its real 4096-record capacity. - self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + // The backing is kept. In every profile the block list declares its real 4096-record + // capacity. + self.ledger.reinit(); self.exhausted_logged = false; + self.reclaim_pending = false; if tuning::PB_GROW { // C1 names the initial pages again. self.c1_written = false; @@ -470,7 +500,7 @@ impl Pools { } let heap = client::PB_HEAP; let initial = self.ledger.initial; - if initial > self.backed() { + if initial > self.ledger.backed { return Err(EINVAL); } write_records( @@ -524,133 +554,248 @@ impl Pools { super::params::pb_max_blocks().clamp(self.ledger.initial, self.ledger.max) } - /// The blocks with backing, from block 0. - pub(crate) fn backed(&self) -> u32 { - self.chunks.last().map_or(0, PbChunk::end) + /// Maps the backed chunks `vm` does not hold yet ([`VmState::map_heap`]). + fn map_heap(&self, vm: &VmState) -> Result { + vm.map_heap(self.chunks.iter().map(PbChunk::gem)) + } + + /// Frees the backing chunks from index `keep` on (their VM mappings are dropped by the + /// caller); the RAM returns when the last mapping of a chunk is gone. + fn free_chunks(&mut self, keep: usize) -> usize { + let freed = self.chunks.len().saturating_sub(keep); + self.chunks.truncate(keep); + self.ledger.unback_to(self.chunks.len() as u32); + freed + } + + /// Backs one more chunk; the caller maps it where needed before the ledger posts from it. + fn back_chunk(&mut self) -> Result> { + if self.chunks.len() >= growth::MAX_CHUNKS as usize { + return Err(ENOSPC); + } + let chunk = PbChunk::new(&self.drm)?; + let gem = chunk.gem(); + self.chunks.push(chunk, GFP_KERNEL)?; + Ok(gem) } /// The debugfs line `pb: blocks initial/posted/firmware-max/max backed blocks/chunks - /// requests grown refused bad`. + /// requests grown refused bad waits refills reclaims freed` (new keys are appended). pub(crate) fn write_stats(&self, f: &mut impl fmt::Write) -> fmt::Result { writeln!( f, - "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {}", + "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {} waits {} refills {} reclaims {} freed {}", self.ledger.initial, self.ledger.posted, self.soft_max(), self.ledger.max, - self.backed(), + self.ledger.backed, self.chunks.len(), self.stats.requests, self.stats.grown, self.stats.refused, - self.stats.bad + self.stats.bad, + self.stats.waits, + self.stats.refills, + self.stats.reclaims, + self.stats.freed ) } } impl Firmware { - /// Maps the PB backing into `vm` up to its last chunk, before anything of a prefix of the - /// VM is published (the start of `publish_prefix_into`). Render work runs in one VM at a - /// time (`Engine::publish`), so the VM about to run is the only one through which the GPU - /// can reach a posted block next; a VM created or idle while the backing grew catches up - /// here, also a closed VM whose accepted jobs still run. + /// Maps the PB backing into `vm`, before anything of a prefix of the VM is published (the + /// start of `publish_prefix_into`). Render work runs in one VM at a time + /// (`Engine::publish`), so the VM about to run is the only one through which the GPU can + /// reach a posted block next; a VM created or idle while the backing grew catches up here, + /// also a closed VM whose accepted jobs still run. A chunk is mapped only once + /// ([`VmState::map_heap`]); one that fails leaves the chunks mapped before it, and the + /// prefix is not published. The leaf is GPU-only, so dropping an alias is a plain unmap. /// - /// Each chunk's alias joins the VM's driver mappings, held until the VM is torn down, and - /// the VM's count moves past the chunk only then. When that fails, the alias just made is - /// dropped again: nothing of the VM runs before its count covers the chunk, and the leaf is - /// GPU-only, so the drop is a plain unmap. The chunks mapped before stay counted. - /// - /// The one-VM rule is checked first: a flight of another VM, or a count past the chunks, - /// fails with EPROTO before anything is mapped. + /// The one-VM rule is checked first: a flight of another VM fails with EPROTO before + /// anything is mapped. pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result { - let mut mapped = vm.pb_chunks.load(Ordering::Relaxed); - if mapped > self.pools.chunks.len() - || self - .engine - .flights - .iter() - .any(|f| !core::ptr::eq(&*f.job.vm, vm)) - { + let others = self + .engine + .flights + .iter() + .filter(|f| !core::ptr::eq(&*f.job.vm, vm)) + .count(); + if others != 0 { dev_err!( self.dev(), - "PB backing: ctx {} maps {} of {} chunks with {} flights of another VM\n", - vm.slot, - mapped, + "PB backing: VM {} maps {} chunks with {} flights of another VM\n", + vm.id, self.pools.chunks.len(), - self.engine - .flights - .iter() - .filter(|f| !core::ptr::eq(&*f.job.vm, vm)) - .count() + others ); return Err(super::boot::eproto()); } - while let Some(chunk) = self.pools.chunks.get(mapped) { - let mut aliases = KVec::with_capacity(1, GFP_KERNEL)?; - aliases.push(chunk.map(&vm.vm)?, GFP_KERNEL)?; - vm.vm.append_driver_mappings(aliases)?; - mapped += 1; - vm.pb_chunks.store(mapped, Ordering::Relaxed); - } - Ok(()) + self.pools.map_heap(vm) } - /// The VM whose render work runs: every flight's (`Engine::publish` puts a flight behind - /// another only for the same VM), and it must be the asking context's. EPROTO otherwise. - fn running_vm(&self, ctx: u32) -> Result> { - let flights = &self.engine.flights; - let vm = flights - .first() - .ok_or_else(super::boot::eproto)? - .job - .vm - .clone(); - if vm.slot != ctx || flights.iter().any(|f| !Arc::ptr_eq(&f.job.vm, &vm)) { - dev_err!( - self.dev(), - "PB growth: ctx {} is not the one VM in flight (ctx {}, {} flights)\n", - ctx, - vm.slot, - flights.len() - ); - return Err(super::boot::eproto()); + /// Backs chunks until the reserve is whole, mapping each into the VMs of the flights (by + /// the one-VM rule, the VM that runs) before the ledger may post from it; any other VM + /// maps it at its next publication ([`Self::map_pb`]). A failure is logged and leaves the + /// reserve short: requests then wait, and the next pass tries again. + pub(crate) fn refill_backing(&mut self) { + // A pending re-initialization is about to drop the posted count: nothing to back for, + // unless a render published before it still runs (the flights behind a fragment kill + // keep asking for growth until they retire). + let render = (0..self.engine.flights.len()).any(|i| self.receipts(i).renders != 0); + if self.recovery.reinit && !render { + return; } - Ok(vm) + while self.pools.ledger.refill_due() { + let result = (|| -> Result { + let gem = self.pools.back_chunk()?; + let i = self.pools.chunks.len() - 1; + for flight in self.engine.flights.iter() { + flight.job.vm.map_chunk(i, gem.clone())?; + } + Ok(()) + })(); + match result { + Ok(()) => { + self.pools.ledger.back_chunk(); + self.pools.stats.refills += 1; + dev_dbg!( + self.dev(), + "PB heap: chunk {} backed ({} of {} blocks posted)\n", + self.pools.chunks.len() - 1, + self.pools.ledger.posted, + self.pools.ledger.backed + ); + } + Err(e) => { + // A chunk that could not be mapped into a flight's VM is not counted: drop + // it (and its aliases in the VMs) again. + let backed = (self.pools.ledger.backed / growth::CHUNK_BLOCKS) as usize; + if self.pools.chunks.len() > backed { + for flight in self.engine.flights.iter() { + flight.job.vm.unmap_heap_above(backed); + } + self.pools.chunks.truncate(backed); + } + dev_warn!( + self.dev(), + "PB heap: backing chunk {} failed: {} ({} of {} blocks posted)\n", + backed, + e.to_errno(), + self.pools.ledger.posted, + self.pools.ledger.backed + ); + return; + } + } + } + } + + /// Notes the engine idle (nothing published, nothing pending) or busy; the idle time is + /// what [`Self::reclaim_check`] measures. Called at the end of every poll pass and + /// publication (the pass that retires the last flight is not re-armed) and at a job's + /// arrival. + pub(crate) fn note_idle(&mut self) { + let idle = self.engine.flights.is_empty() && self.engine.pending.is_empty(); + self.pools.idle.note(idle, super::engine::now_ns()); } - /// Backs the PB up to block `end` in the VM that runs, the asking context `ctx`, before an - /// answer posts those blocks (nothing new when they are backed already): chunks of at most - /// [`growth::CHUNK_BLOCKS`] blocks are allocated ([`object::gpu_backing`]), mapped into that - /// VM and kept, one at a time. The VM first catches up ([`Self::map_pb`], which also checks - /// the one-VM rule), so its count equals the chunks. A chunk is kept - /// only once its alias in the VM is held, and the VM's count moves with it; when the alias - /// cannot be held, the alias and the chunk are dropped, neither ever reachable (the leaf is - /// GPU-only: a plain unmap). An error leaves the chunks backed so far for the next request, - /// and nothing is posted. The allocation sleeps (shmem pages with `__GFP_RETRY_MAYFAIL`) - /// while the firmware waits for the answer; how long is not bounded here. - fn back_pb(&mut self, end: u32, ctx: u32) -> Result { - let vm = self.running_vm(ctx)?; - self.map_pb(&vm)?; - while self.pools.backed() < end { - let (first, blocks) = - growth::next_chunk(self.pools.backed(), self.pools.ledger.max).ok_or(ENOSPC)?; - if self.pools.chunks.len() == self.pools.chunks.capacity() { - return Err(ENOSPC); + /// Reclaims the PB when the engine has been idle for `tuning::PB_IDLE_MS` with more than + /// `tuning::PB_RECLAIM_BLOCKS` posted or more backing than the posted blocks need, or for + /// [`growth::DRAIN_IDLE_NS`] with more than half the maximum posted + /// ([`growth::Ledger::drain_reinit_due`]; re-initialization only): the next render prefix + /// re-initializes the render allocators and the PB (InitBM of the initial blocks; + /// `recovery.reinit`, as after a recovered abort but with no halt), and with + /// `tuning::PB_RECLAIM` 2 the backing chunks above those of the initial posting and the reserve + /// are freed now, unmapped from every VM of the registry (a VM that publishes later drops + /// its stale mappings at [`VmState::map_heap`]). While a reclaim's re-initialization is + /// pending (a client-boundary one kept the backing), `tuning::PB_RECLAIM` 2 frees the backing + /// above those chunks once the engine has been idle for `tuning::PB_IDLE_MS` + /// ([`growth::Ledger::pending_free_due`]). Called under the device lock when a job arrives + /// at an idle engine and in idle poll passes. + pub(crate) fn reclaim_check(&mut self) { + let level = super::params::pb_reclaim(); + if level == 0 + || !self.engine.flights.is_empty() + || !self.engine.pending.is_empty() + || (self.recovery.reinit && !self.pools.reclaim_pending) + || self.recovery.armed.is_some() + || self.engine.frozen + || self.recovery.failed().is_some() + { + return; + } + let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); + let floor = super::params::pb_reclaim_blocks(); + let min_idle = super::params::pb_idle_ns(); + let ledger = &self.pools.ledger; + if self.recovery.reinit { + if level >= 2 && ledger.pending_free_due(idle_ns, min_idle) { + let posted = ledger.posted; + let freed = self.free_backing(); + dev_info!( + self.dev(), + "PB heap: idle {} ms with {} blocks posted and the re-initialization pending: {} chunks freed, {} backed\n", + idle_ns / 1_000_000, + posted, + freed, + self.pools.chunks.len() + ); } - let chunk = PbChunk::new(&self.pools.drm, first, blocks)?; - let mut aliases = KVec::with_capacity(1, GFP_KERNEL)?; - aliases.push(chunk.map(&vm.vm)?, GFP_KERNEL)?; - vm.vm.append_driver_mappings(aliases)?; - // Within the capacity checked above. - self.pools - .chunks - .push_within_capacity(chunk) - .map_err(|_| EIO)?; - vm.pb_chunks - .store(self.pools.chunks.len(), Ordering::Relaxed); + return; } - Ok(()) + let idle = ledger.reclaim_due(floor, idle_ns, min_idle) + || (level >= 2 && ledger.free_due(idle_ns, min_idle)); + if !idle && !ledger.drain_reinit_due(idle_ns) { + return; + } + self.recovery.reinit = true; + self.pools.reclaim_pending = true; + self.pools.stats.reclaims += 1; + let posted = self.pools.ledger.posted; + let mut freed = 0; + // A short idle keeps the backing: the next scene would allocate it again. + if level >= 2 && idle { + freed = self.free_backing(); + } + dev_info!( + self.dev(), + "PB heap: idle {} ms with {} blocks posted: re-initialization to {} at the next render, {} chunks freed, {} backed\n", + idle_ns / 1_000_000, + posted, + self.pools.ledger.initial, + freed, + self.pools.chunks.len() + ); + } + + /// Frees the backing chunks above those a re-initialized heap keeps + /// ([`growth::Ledger::keep_chunks`]) and drops their mappings in every VM that could + /// publish; returns the chunks freed. If the VMs cannot be walked, their mappings keep the + /// pages until each VM drops its stale aliases ([`VmState::map_heap`]): nothing is counted + /// freed. + fn free_backing(&mut self) -> usize { + let keep = self.pools.ledger.keep_chunks() as usize; + let dropped = self.pools.free_chunks(keep); + // The mappings hold the chunks' pages: drop them in every VM that could publish. + let freed = match self.vms.open() { + Ok(states) => { + for state in states.iter() { + state.unmap_heap_above(keep); + } + dropped + } + Err(e) => { + dev_warn!( + self.dev(), + "PB heap: reclaim could not walk the VMs: {}\n", + e.to_errno() + ); + 0 + } + }; + self.pools.stats.freed += freed as u64; + freed } /// (Re)initializes the render UMA managers, PB lists and PB manager. `rebuild`: the priming @@ -683,9 +828,12 @@ impl Firmware { /// DEVCTL 7 reports success; at the ledger's maximum the reply is the refusal (+4 = 0), /// which gives the firmware no new pages, and after two refusals requests are left /// unanswered. The firmware imports the blocks itself: P is never rewritten. Only the - /// requests queued before the pass are taken. A failed reply is logged; a request that - /// cannot be processed at all (a firmware object access failed) is logged and left - /// unanswered like an invalid one. Never fails: the poll pass continues. + /// requests queued before the pass are taken. A request with no backed block left waits: + /// it goes back to the head of the queue for the next pass, and this pass ends its answers + /// there. After the answers the reserve is refilled ([`Self::refill_backing`]). A failed + /// reply is logged; a request that cannot be processed at all (a firmware object access + /// failed) is logged and left unanswered like an invalid one. Never fails: the poll pass + /// continues. pub(crate) fn answer_growth(&mut self) -> Result { // Only the requests queued before this pass: a reply's wait drains the rings and may // queue the next request, which the next pass answers (the device lock is released in @@ -695,23 +843,67 @@ impl Firmware { let Ok(request) = self.events.grow.remove(0) else { break; }; - self.pools.stats.requests += 1; - if let Err(e) = self.answer_request(&request) { - self.pools.stats.bad += 1; - dev_err!( - self.dev(), - "PB growth request {} ctx {} not answered: {}\n", - request[buffer::grow_request::COUNTER], - request[buffer::grow_request::CONTEXT], - e.to_errno() - ); + match self.answer_request(&request) { + Ok(true) => self.pools.stats.requests += 1, + Ok(false) => { + // Waits for backing: back at the head, counted once answered. + self.pools.stats.waits += 1; + // The queue has room: this request was just taken from it. + if self.events.grow.insert_within_capacity(0, request).is_err() { + self.pools.stats.bad += 1; + } + break; + } + Err(e) => { + self.pools.stats.requests += 1; + self.pools.stats.bad += 1; + dev_err!( + self.dev(), + "PB growth request {} ctx {} not answered: {}\n", + request[buffer::grow_request::COUNTER], + request[buffer::grow_request::CONTEXT], + e.to_errno() + ); + } } } + self.refill_backing(); Ok(()) } - /// Processes one growth request. - fn answer_request(&mut self, request: &[u32; 14]) -> Result { + /// How long the idle engine must stay idle before [`Self::reclaim_check`] acts, if it can + /// act without new work ([`growth::Ledger::reclaim_delay`], or with a reclaim's + /// re-initialization pending [`growth::Ledger::pending_free_delay`]); `None` while busy, + /// while a recovery or its re-initialization is pending, or when nothing would become due. + /// An idle engine runs no poll pass by itself, so the poll work is armed for this delay. + pub(crate) fn reclaim_delay_ns(&self) -> Option { + if !self.engine.flights.is_empty() + || !self.engine.pending.is_empty() + || (self.recovery.reinit && !self.pools.reclaim_pending) + || self.recovery.armed.is_some() + || self.engine.frozen + || self.recovery.failed().is_some() + { + return None; + } + let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); + if self.recovery.reinit { + return self.pools.ledger.pending_free_delay( + super::params::pb_reclaim(), + idle_ns, + super::params::pb_idle_ns(), + ); + } + self.pools.ledger.reclaim_delay( + super::params::pb_reclaim(), + super::params::pb_reclaim_blocks(), + idle_ns, + super::params::pb_idle_ns(), + ) + } + + /// Processes one growth request: `Ok(false)` when it waits for backing. + fn answer_request(&mut self, request: &[u32; 14]) -> Result { let ctl = self.init.control.clone().ok_or(ENODEV)?; let epoch = ctl.r32(control::HALT_COUNT)? as u64 | (ctl.r32(control::HALT_COUNT + 4)? as u64) << 32; @@ -721,7 +913,7 @@ impl Firmware { .flights .iter() .enumerate() - .any(|(i, f)| self.receipts(i).renders != 0 && f.job.vm.slot == ctx); + .any(|(i, f)| self.receipts(i).renders != 0 && f.job.slot() == ctx); // At the maximum, after the refusals, the request is left unanswered: the poll pass // returns, and the host deadline and its recovery path apply to a stalled TA. if self.pools.ledger.exhausted() { @@ -737,7 +929,7 @@ impl Firmware { self.pools.ledger.max ); } - return Ok(()); + return Ok(true); } // A request from before a halt has no authority over the PB now. if !self @@ -758,15 +950,15 @@ impl Firmware { live as u32, tuning::PB_GROW as u32 ); - return Ok(()); + return Ok(true); } - // Every block the answer posts is backed in the VM that runs before anything of the - // answer is written; the one-VM rule is checked for every answer. If that fails, nothing - // is posted or answered (answer_growth logs it; the deadline applies), and the backing - // made so far stays for the next request. - let end = self.pools.ledger.posted + self.pools.ledger.next_count(); - self.back_pb(end, ctx)?; - let (first, n) = self.pools.ledger.accept(request); + // A grant posts only backed blocks, and every backed chunk is mapped in the VM of the + // flights (at its publication, or at the refill that backed it). + let (first, n) = match self.pools.ledger.accept(request) { + Answer::Grant { first, count } => (first, count), + Answer::Refuse => (self.pools.ledger.posted, 0), + Answer::Wait => return Ok(false), + }; let heap = client::PB_HEAP; for b in first..first + n { self.pools.pb_blocks.w64( @@ -831,6 +1023,6 @@ impl Firmware { ret ); } - Ok(()) + Ok(true) } } diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index 20b298e641459..e9eaee242ac51 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -12,6 +12,15 @@ //! its recovery path then apply, without a guarantee that they end the stall. A request from //! before a halt (epoch) or not matching a published render prefix is left unanswered. //! +//! The heap's VA range covers [`MAX_BLOCKS`] blocks; its RAM comes in chunks of +//! [`CHUNK_BLOCKS`] blocks from the heap base, which the ledger counts as `backed`. A request is +//! granted from the backed blocks above `posted`. A chunk is due ([`Ledger::refill_due`]) +//! whenever fewer than [`RESERVE_BLOCKS`] backed blocks remain above the posted count, so that +//! it can be backed after a reply; a request that finds no backed block waits +//! ([`Answer::Wait`]) and keeps its counter. A re-initialization ([`Ledger::reinit`]) returns +//! the posted count to the initial size and keeps the backing; [`Ledger::keep_chunks`] gives the +//! chunks that the initial posting and the reserve need. +//! //! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. use crate::g15::fw::buffer::{grow_request, BLOCK_LIST_RECORDS, GROW_STEP, MAX_BLOCKS, MIN_BLOCKS}; @@ -26,6 +35,8 @@ pub(crate) struct Ledger { pub(crate) max: u32, /// Blocks posted to the block list, cumulative (the block_ctl+4 producer is this modulo K). pub(crate) posted: u32, + /// Blocks with RAM behind them, from the heap base. + pub(crate) backed: u32, /// The next expected request counter. pub(crate) next_request: u32, /// A refusal at the maximum was logged since the last initialization. @@ -41,26 +52,30 @@ pub(crate) struct Ledger { /// Refusals answered at the maximum before requests are left to the deadline. pub(crate) const MAX_REFUSALS: u32 = 2; -/// Blocks of each PB backing chunk after the initial one (4 MiB): growth backs at most this -/// much at a time while the firmware waits for its answer. -pub(crate) const CHUNK_BLOCKS: u32 = 32; - -/// The backing chunk that follows `backed` blocks of a ledger with maximum `max`: its first block -/// and size, at most [`CHUNK_BLOCKS`] and never past `max`; none from `max` on. -pub(crate) const fn next_chunk(backed: u32, max: u32) -> Option<(u32, u32)> { - if backed >= max { - None - } else if max - backed < CHUNK_BLOCKS { - Some((backed, max - backed)) - } else { - Some((backed, CHUNK_BLOCKS)) - } -} +/// Blocks of each PB backing chunk (16 MiB). +pub(crate) const CHUNK_BLOCKS: u32 = 128; + +/// Backed blocks kept above the posted count: at 1-2 ms per request of ten blocks, 13 requests +/// of lead for the allocation of the next chunk. +pub(crate) const RESERVE_BLOCKS: u32 = CHUNK_BLOCKS; -/// The chunks that back a ledger with `initial` and `max` blocks at most: the initial one and -/// [`next_chunk`]'s from `initial` to `max`. -pub(crate) const fn max_chunks(initial: u32, max: u32) -> usize { - 1 + max.saturating_sub(initial).div_ceil(CHUNK_BLOCKS) as usize +/// Chunks of the whole heap. +pub(crate) const MAX_CHUNKS: u32 = MAX_BLOCKS / CHUNK_BLOCKS; + +/// The idle time before a re-initialization past half the maximum (a client boundary, not a +/// gap between one client's renders). +pub(crate) const DRAIN_IDLE_NS: u64 = 50_000_000; + +/// The answer to an admitted growth request. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Answer { + /// Post blocks `[first, first + count)` (`count` at most [`GROW_STEP`]) and reply grown. + Grant { first: u32, count: u32 }, + /// Reply refused: the maximum is posted. + Refuse, + /// No backed block above the posted count: leave the request queued until a chunk is + /// backed. + Wait, } /// The most blocks a ledger posts: the mapped heap, and below K, so that the block_ctl+4 @@ -85,17 +100,141 @@ impl Ledger { initial, max, posted: initial, + backed: 0, next_request: 0, refusal_logged: false, refusals: 0, } } + /// A re-initialization: the initial blocks are posted again, the request counter and the + /// refusals start over, the backing stays. + pub(crate) const fn reinit(&mut self) { + self.posted = self.initial; + self.next_request = 0; + self.refusal_logged = false; + self.refusals = 0; + } + /// The live block_ctl+4 producer. pub(crate) const fn producer(&self) -> u32 { self.posted % BLOCK_LIST_RECORDS } + /// Records one more backed chunk. + pub(crate) const fn back_chunk(&mut self) { + self.backed += CHUNK_BLOCKS; + } + + /// Drops the backing to `chunks` chunks. + pub(crate) const fn unback_to(&mut self, chunks: u32) { + self.backed = chunks * CHUNK_BLOCKS; + } + + /// Backed blocks above the posted count. + pub(crate) const fn spare(&self) -> u32 { + self.backed.saturating_sub(self.posted) + } + + /// Whether another chunk is due: fewer than the reserve above the posted count and the heap + /// not yet backed whole. + pub(crate) const fn refill_due(&self) -> bool { + self.backed < MAX_BLOCKS && self.spare() < RESERVE_BLOCKS + } + + /// Chunks that `blocks` posted blocks and the reserve need. + pub(crate) const fn chunks_for(blocks: u32) -> u32 { + let n = (blocks + RESERVE_BLOCKS).div_ceil(CHUNK_BLOCKS); + if n > MAX_CHUNKS { + MAX_CHUNKS + } else { + n + } + } + + /// Chunks a re-initialized heap keeps: those of the initial posting and the reserve. + pub(crate) const fn keep_chunks(&self) -> u32 { + Self::chunks_for(self.initial) + } + + /// Whether the heap is to be re-initialized: idle for `idle_ns` with more than `floor` + /// blocks posted, and grown since the initialization (with nothing grown a + /// re-initialization posts the same blocks again). + pub(crate) const fn reclaim_due(&self, floor: u32, idle_ns: u64, min_idle_ns: u64) -> bool { + self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns + } + + /// Whether the backing is to be freed: idle for `idle_ns` with more chunks backed than the + /// posted blocks and the reserve need. A client-boundary re-initialization keeps the + /// backing of a heavy scene; if the next client posts below the reclaim floor, this frees it. + pub(crate) const fn free_due(&self, idle_ns: u64, min_idle_ns: u64) -> bool { + idle_ns >= min_idle_ns && self.backed > Self::chunks_for(self.posted) * CHUNK_BLOCKS + } + + /// While a reclaim's re-initialization is pending (the posted count returns to the initial + /// size at the next render): whether the backing above [`Self::keep_chunks`] is to be + /// freed, idle for `min_idle_ns`. A client-boundary re-initialization keeps the backing; + /// with no render after it, this frees it. + pub(crate) const fn pending_free_due(&self, idle_ns: u64, min_idle_ns: u64) -> bool { + idle_ns >= min_idle_ns && self.backed > self.keep_chunks() * CHUNK_BLOCKS + } + + /// How much longer the engine must stay idle, from `idle_ns`, before + /// [`Self::pending_free_due`] with `level`; `None` when it cannot become due (level below + /// 2, or nothing above the kept chunks). 0 when it is due now. + pub(crate) const fn pending_free_delay( + &self, + level: u32, + idle_ns: u64, + min_idle_ns: u64, + ) -> Option { + if level >= 2 && self.backed > self.keep_chunks() * CHUNK_BLOCKS { + Some(min_idle_ns.saturating_sub(idle_ns)) + } else { + None + } + } + + /// Whether a short idle re-initializes the heap: more than half the maximum posted and + /// idle for [`DRAIN_IDLE_NS`]. Back-to-back heavy scenes (a new client every few hundred + /// ms) never reach `tuning::PB_IDLE_MS`, and posted blocks never come back while a PB lives, so + /// without this they run the heap to the maximum. A re-initialization with no idle time + /// faulted at the heap base (at a client switch, the second in a row with no recovery + /// between); with 50 ms of idle none did, also between two renders of one client. + pub(crate) const fn drain_reinit_due(&self, idle_ns: u64) -> bool { + self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS + } + + /// How much longer the engine must stay idle, from `idle_ns`, before a reclaim with + /// `level` is due ([`Self::reclaim_due`] with `floor` and `min_idle_ns`, [`Self::free_due`] + /// at level 2, or [`Self::drain_reinit_due`]); `None` when none can become due while the + /// ledger stays as it is. 0 when one is due now. + pub(crate) const fn reclaim_delay( + &self, + level: u32, + floor: u32, + idle_ns: u64, + min_idle_ns: u64, + ) -> Option { + if level == 0 { + return None; + } + let grown = self.posted > self.initial; + let mut due = u64::MAX; + if grown && self.posted > self.max / 2 { + due = DRAIN_IDLE_NS; + } + let free = level >= 2 && self.backed > Self::chunks_for(self.posted) * CHUNK_BLOCKS; + if ((grown && self.posted > floor) || free) && min_idle_ns < due { + due = min_idle_ns; + } + if due == u64::MAX { + None + } else { + Some(due.saturating_sub(idle_ns)) + } + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -114,21 +253,24 @@ impl Ledger { && request_epoch(request) == epoch } - /// The blocks the next accepted request posts: at most [`GROW_STEP`], none at the maximum. - pub(crate) fn next_count(&self) -> u32 { - GROW_STEP.min(self.max - self.posted) - } - - /// Accepts an admitted request: the next expected counter follows it, and the returned - /// blocks `[first, first + count)` are to be written to the block list, `count` 0 meaning a - /// refusal at the maximum. - pub(crate) fn accept(&mut self, request: &[u32; 14]) -> (u32, u32) { + /// Accepts an admitted request. Granted (at most [`GROW_STEP`] blocks, and no more than are + /// backed) or refused at the maximum, the next expected counter follows it. A request that + /// finds no backed block waits: its counter is not consumed, and it is accepted again once a + /// chunk is backed. + pub(crate) fn accept(&mut self, request: &[u32; 14]) -> Answer { + let room = self.max - self.posted; + if room != 0 && self.spare() == 0 { + return Answer::Wait; + } self.next_request = request[grow_request::COUNTER].wrapping_add(1); - let count = self.next_count(); - if count == 0 { + if room == 0 { self.refusals += 1; + return Answer::Refuse; + } + Answer::Grant { + first: self.posted, + count: GROW_STEP.min(room).min(self.spare()), } - (self.posted, count) } /// Records `count` blocks as posted after their records were written and returns the new @@ -154,6 +296,37 @@ impl Ledger { } } +/// The engine's idle clock for the reclaim policy: when nothing was published and nothing +/// pending, from the pass that retired the last flight on. Noted at the end of every poll pass +/// and publication (a pass that empties the flights is not re-armed, so a note only at the +/// start of a pass with empty flights depends on a backup pass happening to run) and, busy, +/// when a job is enqueued. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct IdleClock { + /// When the idle began; 0 while busy. + since_ns: u64, +} + +impl IdleClock { + /// Notes the engine idle or busy at `now_ns`; an idle already begun keeps its start. + pub(crate) fn note(&mut self, idle: bool, now_ns: u64) { + if !idle { + self.since_ns = 0; + } else if self.since_ns == 0 { + self.since_ns = now_ns.max(1); + } + } + + /// The idle time at `now_ns`; 0 while busy. + pub(crate) fn idle_ns(&self, now_ns: u64) -> u64 { + if self.since_ns == 0 { + 0 + } else { + now_ns.wrapping_sub(self.since_ns) + } + } +} + /// The halt epoch a request names (u64 at words 5..6). pub(crate) const fn request_epoch(request: &[u32; 14]) -> u64 { request[grow_request::EPOCH] as u64 | (request[grow_request::EPOCH + 1] as u64) << 32 @@ -185,12 +358,19 @@ mod tests { q } + /// A ledger with the heap backed whole. + fn backed(grow: bool, blocks: u32, max_blocks: u32) -> Ledger { + let mut l = Ledger::new(grow, blocks, max_blocks); + l.backed = MAX_BLOCKS; + l + } + #[test] fn clamps() { - let l = Ledger::new(true, 1024, 2048); + let l = backed(true, 1024, 2048); assert_eq!( - (l.initial, l.max, l.posted, l.next_request), - (1024, 2048, 1024, 0) + (l.initial, l.max, l.posted, l.backed, l.next_request), + (1024, 2048, 1024, 4096, 0) ); assert_eq!(l.producer(), 1024); assert_eq!(Ledger::new(true, 10, 20).initial, 64); @@ -198,13 +378,14 @@ mod tests { assert_eq!(Ledger::new(true, 5000, 100).initial, 4095); assert_eq!(Ledger::new(true, 512, 1024).max, 1024); assert_eq!(Ledger::new(true, 512, 100).max, 512); + assert_eq!(Ledger::new(true, 512, 100).backed, 0); let fixed = Ledger::new(false, 512, 1024); assert_eq!((fixed.initial, fixed.max, fixed.posted), (4095, 4095, 4095)); - // The values the driver passes (tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS) are kept. - let l = Ledger::new(true, 1024, 4095); + // The values the driver passes (tuning::PB_BLOCKS 64, tuning::PB_MAX_BLOCKS) are kept. + let l = Ledger::new(true, 64, 4095); assert_eq!( (l.initial, l.max, l.posted, l.producer()), - (1024, 4095, 1024, 1024) + (64, 4095, 64, 64) ); } @@ -235,18 +416,19 @@ mod tests { for grow in [true, false] { for blocks in values { for max_blocks in values { - let mut l = Ledger::new(grow, blocks, max_blocks); + let mut l = backed(grow, blocks, max_blocks); assert!(MIN_BLOCKS <= l.initial && l.initial <= l.max && l.max <= MAX_POSTED); assert_eq!(l.posted, l.initial); assert!((1..=MAX_POSTED).contains(&l.producer())); // Every producer of a growth to the maximum stays in 1..=4095. let mut counter = 0; loop { - let (first, n) = l.accept(&request(5, counter, 1)); + let (first, n) = match l.accept(&request(5, counter, 1)) { + Answer::Grant { first, count } => (first, count), + Answer::Refuse => break, + Answer::Wait => panic!("backed whole"), + }; assert_eq!(first, l.posted); - if n == 0 { - break; - } let producer = l.post(n); assert!((1..=MAX_POSTED).contains(&producer)); assert_eq!(producer, l.posted); @@ -261,7 +443,7 @@ mod tests { #[test] fn admission() { - let l = Ledger::new(true, 1024, 2048); + let l = backed(true, 1024, 2048); let q = request(5, 0, 3); assert!(l.admits(&q, true, true, 3)); assert!(!l.admits(&q, false, true, 3)); @@ -283,17 +465,20 @@ mod tests { #[test] fn growth_to_the_maximum() { // 512 initial and 1024 maximum blocks. - let mut l = Ledger::new(true, 512, 1024); + let mut l = backed(true, 512, 1024); let mut grown = 0; let mut counter = 0; loop { let q = request(5, counter, 1); assert!(l.admits(&q, true, true, 1)); - let (first, n) = l.accept(&q); - assert_eq!(first, 512 + grown); - if n == 0 { - break; - } + let n = match l.accept(&q) { + Answer::Grant { first, count } => { + assert_eq!(first, 512 + grown); + count + } + Answer::Refuse => break, + Answer::Wait => panic!("backed whole"), + }; assert!(n <= 10); assert_eq!(l.post(n), (512 + grown + n) % 4096); grown += n; @@ -310,26 +495,97 @@ mod tests { assert_eq!(l.refusals, 1); assert!(!l.exhausted()); let q = request(5, counter + 3, 1); - assert_eq!(l.accept(&q), (1024, 0)); + assert_eq!(l.accept(&q), Answer::Refuse); assert!(l.exhausted()); assert!(!l.admits(&request(5, counter + 4, 1), true, true, 1)); - // A fresh ledger starts over. + // A fresh ledger starts over, and so does a re-initialized one (backing kept). assert!(!Ledger::new(true, 512, 1024).exhausted()); + l.reinit(); + assert_eq!( + (l.posted, l.next_request, l.refusals, l.backed), + (512, 0, 0, 4096) + ); + assert!(!l.exhausted() && !l.refusal_logged); } - /// Backing for every ledger the constructor makes, the driver's included: the chunks from - /// the initial one are contiguous, at most [`CHUNK_BLOCKS`] each, end at the maximum and - /// number at most [`max_chunks`]; growth with the backing made first never posts a block - /// past it. #[test] - fn backing_chunks() { - assert_eq!(max_chunks(1024, 4095), 1 + 96); - assert_eq!(max_chunks(MIN_BLOCKS, MAX_POSTED), 1 + 126); - assert_eq!(max_chunks(MAX_POSTED, MAX_POSTED), 1); - assert_eq!(next_chunk(1024, 4095), Some((1024, 32))); - assert_eq!(next_chunk(4064, 4095), Some((4064, 31))); - assert_eq!(next_chunk(4095, 4095), None); - assert_eq!(next_chunk(4096, 4095), None); + fn producer_wraps_at_k() { + let mut l = backed(true, 2048, 2048); + assert_eq!(l.producer(), 2048); + l.posted = 4095; + assert_eq!(l.post(1), 0); + assert_eq!(l.post(10), 10); + } + + #[test] + fn backing() { + // 64 initial blocks: two chunks (the initial posting and the reserve). + assert_eq!(Ledger::chunks_for(64), 2); + assert_eq!(Ledger::chunks_for(1024), 9); + assert_eq!(Ledger::chunks_for(4095), MAX_CHUNKS); + assert_eq!(MAX_CHUNKS, 32); + let mut l = Ledger::new(true, 64, 4095); + assert_eq!(l.keep_chunks(), 2); + assert_eq!(l.spare(), 0); + assert!(l.refill_due()); + // Nothing backed: a request waits and the counter does not advance. + let q = request(3, 0, 1); + assert_eq!(l.accept(&q), Answer::Wait); + assert_eq!(l.next_request, 0); + l.back_chunk(); + l.back_chunk(); + assert_eq!((l.backed, l.spare()), (256, 192)); + assert!(!l.refill_due()); + // Granting until the reserve line, then a refill is due; at the end of the backing the + // grant is clipped and the next request waits. + assert_eq!( + l.accept(&q), + Answer::Grant { + first: 64, + count: 10 + } + ); + assert_eq!(l.post(10), 74); + assert_eq!(l.next_request, 1); + l.posted = 128; + assert!(!l.refill_due()); + l.posted = 129; + assert!(l.refill_due()); + l.posted = 250; + assert_eq!( + l.accept(&request(3, 1, 1)), + Answer::Grant { + first: 250, + count: 6 + } + ); + l.post(6); + assert_eq!(l.accept(&request(3, 2, 1)), Answer::Wait); + // The heap end: no refill past the whole heap, the maximum refuses. + l.backed = MAX_BLOCKS; + l.posted = 4090; + assert!(!l.refill_due()); + assert_eq!( + l.accept(&request(3, 2, 1)), + Answer::Grant { + first: 4090, + count: 5 + } + ); + l.post(5); + assert_eq!(l.accept(&request(3, 3, 1)), Answer::Refuse); + // Re-initialization keeps the backing; dropping it to the kept chunks. + l.reinit(); + assert_eq!((l.posted, l.backed), (64, 4096)); + l.unback_to(l.keep_chunks()); + assert_eq!(l.backed, 256); + } + + /// Every ledger the constructor makes, the driver's included, backed a chunk at a time + /// whenever the reserve is short: no grant posts a block past the backing, no request waits, + /// and the backing ends within the heap. + #[test] + fn backing_ahead_of_every_grant() { let values = [ 0, 1, MIN_BLOCKS, 65, 1023, 1024, 1025, 3072, 4094, MAX_POSTED, 5000, ]; @@ -337,40 +593,308 @@ mod tests { for blocks in values { for max_blocks in values { let mut l = Ledger::new(grow, blocks, max_blocks); - let (mut backed, mut chunks) = (l.initial, 1); + for _ in 0..Ledger::chunks_for(l.initial) { + l.back_chunk(); + } let mut counter = 0; loop { - let end = l.posted + l.next_count(); - while backed < end { - let (first, n) = next_chunk(backed, l.max).unwrap(); - assert_eq!(first, backed); - assert!(n >= 1 && n <= CHUNK_BLOCKS); - backed += n; - chunks += 1; - } - let (first, n) = l.accept(&request(5, counter, 1)); - assert!(first + n <= backed); - if n == 0 { - break; + match l.accept(&request(5, counter, 1)) { + Answer::Grant { first, count } => { + assert_eq!(first, l.posted); + assert!(count >= 1 && first + count <= l.backed); + l.post(count); + } + Answer::Refuse => break, + Answer::Wait => panic!("waited at {} of {}", l.posted, l.backed), } - l.post(n); counter += 1; + while l.refill_due() { + l.back_chunk(); + } } - assert_eq!((l.posted, backed), (l.max, l.max)); - assert!(chunks <= max_chunks(l.initial, l.max)); - assert_eq!(next_chunk(backed, l.max), None); + assert_eq!(l.posted, l.max); + assert!(l.max <= l.backed && l.backed <= MAX_BLOCKS); } } } } #[test] - fn producer_wraps_at_k() { - let mut l = Ledger::new(true, 2048, 2048); - assert_eq!(l.producer(), 2048); - l.posted = 4095; - assert_eq!(l.post(1), 0); - assert_eq!(l.post(10), 10); + fn reclaim() { + let mut l = Ledger::new(true, 64, 4095); + let s = 1_000_000_000; + assert!(!l.reclaim_due(1024, 5 * s, s)); + l.posted = 1024; + assert!(!l.reclaim_due(1024, 5 * s, s)); + l.posted = 1025; + assert!(l.reclaim_due(1024, s, s)); + assert!(!l.reclaim_due(1024, s - 1, s)); + // The clock: busy is 0, an idle keeps its start, busy ends it. + let mut c = IdleClock::default(); + assert_eq!(c.idle_ns(10 * s), 0); + c.note(true, 10 * s); + c.note(true, 12 * s); + assert_eq!(c.idle_ns(15 * s), 5 * s); + c.note(false, 16 * s); + assert_eq!(c.idle_ns(20 * s), 0); + // A clock started at 0 ns still counts as idle. + c.note(true, 0); + assert_eq!(c.idle_ns(s), s - 1); + // Nothing grown since the initialization: no re-initialization, whatever the floor. + let l = Ledger::new(true, 64, 4095); + assert!( + !l.reclaim_due(32, 5 * s, s), + "re-initialization with nothing grown" + ); + } + + /// Heavy scenes of ~1420 blocks each, new clients back to + /// back. The engine re-initializes at a client boundary once past half the maximum, so the + /// heap never reaches it; the backing stays (no refill on the next scene). + #[test] + fn back_to_back_heavy_scenes() { + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS)); + let mut counter = 0; + for _ in 0..10 { + if l.drain_reinit_due(DRAIN_IDLE_NS) { + l.reinit(); + counter = l.next_request; + } + let start = l.posted; + while l.posted < start + 1420 { + while l.refill_due() { + l.back_chunk(); + } + match l.accept(&request(1, counter, 0)) { + Answer::Grant { count, .. } => { + l.post(count); + } + other => panic!("{other:?} at {}", l.posted), + } + counter += 1; + } + assert!(l.posted < l.max); + } + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + assert_eq!(l.backed, 24 * CHUNK_BLOCKS); + l.posted = 2047; + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS)); + l.posted = 2048; + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + // Not between two renders of one client. + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS - 1)); + assert!(!l.drain_reinit_due(0)); + // An initial posting above half the maximum: nothing grew, nothing to drain. + let l = Ledger::new(true, 3000, 4095); + assert!( + !l.drain_reinit_due(DRAIN_IDLE_NS), + "drain with nothing grown" + ); + } + + /// A client-boundary re-initialization keeps the backing + /// of a heavy scene, the next client posts little; once idle, the backing above what the + /// posted blocks need is due to be freed although the posted count is below the floor. + #[test] + fn kept_backing_freed_at_idle() { + let s = 1_000_000_000; + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 2800; + assert!(!l.free_due(5 * s, s)); + l.reinit(); + l.posted = 264; + assert!(!l.reclaim_due(1024, 5 * s, s)); + assert!(!l.free_due(s - 1, s)); + assert!(l.free_due(s, s)); + // After the re-initialization and the free, nothing more is due. + l.reinit(); + l.unback_to(l.keep_chunks()); + assert!(!l.free_due(5 * s, s)); + // A heap backed for what it posts plus the reserve is not freed. + l.posted = 1000; + l.unback_to(Ledger::chunks_for(1000)); + assert!(!l.free_due(5 * s, s)); + } + + /// A heavy scene, a reclaim that frees the backing, refills during the next heavy scene, + /// an idle gap with a small job in it, and the next render must reclaim again. + #[test] + fn reclaim_after_free_then_refill() { + let s = 1_000_000_000; + let (floor, min_idle) = (1024, s); + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + let mut clock = IdleClock::default(); + // A heavy scene grows the PB to 1484 blocks, refilling as it goes. + let heavy = |l: &mut Ledger, clock: &mut IdleClock, t: u64| { + clock.note(false, t); + let mut counter = l.next_request; + while l.posted < 1484 { + while l.refill_due() { + l.back_chunk(); + } + match l.accept(&request(1, counter, 0)) { + Answer::Grant { count, .. } => { + l.post(count); + } + other => panic!("{other:?} at {}", l.posted), + } + counter += 1; + } + // The pass that retires the last flight notes the idle; no later pass may run. + clock.note(true, t + s); + }; + heavy(&mut l, &mut clock, 100 * s); + assert_eq!((l.posted, l.backed), (1484, 13 * CHUNK_BLOCKS)); + // 5 s later a small job arrives: the reclaim is due; it frees the backing above the + // kept chunks; the re-initialization happens at the next render. + let t = 106 * s; + assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); + l.unback_to(l.keep_chunks()); + clock.note(false, t); + assert_eq!(l.backed, 2 * CHUNK_BLOCKS); + clock.note(true, t + s / 2); + // The next render re-initializes, then the second heavy scene refills to 1484 again. + l.reinit(); + assert_eq!((l.posted, l.backed), (64, 256)); + heavy(&mut l, &mut clock, 110 * s); + assert_eq!((l.posted, l.backed), (1484, 13 * CHUNK_BLOCKS)); + // The same gap again: the reclaim must be due at the small job's arrival. + let t = 116 * s; + assert_eq!(clock.idle_ns(t), 5 * s); + assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); + } + + /// The delay names exactly the idle time at which the reclaim check first acts: nothing is + /// due one nanosecond before it, something is due at it, and nothing ever becomes due + /// when it is `None`. + #[test] + fn reclaim_delay_matches_the_checks() { + let s = 1_000_000_000; + let due = |l: &Ledger, level: u32, floor: u32, idle: u64, min_idle: u64| { + level != 0 + && (l.reclaim_due(floor, idle, min_idle) + || (level >= 2 && l.free_due(idle, min_idle)) + || l.drain_reinit_due(idle)) + }; + for level in 0..=2 { + for floor in [0, 64, 1024, 4095] { + for min_idle in [DRAIN_IDLE_NS / 2, 100_000_000, s] { + for posted in [64, 200, 1024, 1025, 2047, 2048, 2900, 4095] { + for chunks in [2, 3, 9, 23, MAX_CHUNKS] { + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(chunks); + l.posted = posted; + match l.reclaim_delay(level, floor, 0, min_idle) { + None => { + for idle in [0, DRAIN_IDLE_NS, min_idle, 100 * s] { + assert!(!due(&l, level, floor, idle, min_idle)); + } + } + Some(d) => { + assert!(due(&l, level, floor, d, min_idle)); + assert!(d == 0 || !due(&l, level, floor, d - 1, min_idle)); + // Counted from an idle already begun. + assert_eq!( + l.reclaim_delay(level, floor, d / 2, min_idle), + Some(d - d / 2) + ); + assert_eq!( + l.reclaim_delay(level, floor, d + s, min_idle), + Some(0) + ); + } + } + } + } + } + } + } + // The case of the J516C run: a client-boundary re-initialization left 23 chunks for + // 264 posted blocks; the free is due after tuning::PB_IDLE_MS of idle. + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 264; + assert_eq!(l.reclaim_delay(2, 1024, 0, s), Some(s)); + assert_eq!(l.reclaim_delay(1, 1024, 0, s), None); + assert_eq!(l.reclaim_delay(0, 1024, 0, s), None); + // A desktop at the initial posting with the kept chunks: nothing to wait for. + l.unback_to(l.keep_chunks()); + l.posted = 64; + assert_eq!(l.reclaim_delay(2, 1024, 0, s), None); + // Nothing grown, with an initial posting above half the maximum or above the floor: + // nothing armed. + let mut l = Ledger::new(true, 3000, 4095); + l.unback_to(Ledger::chunks_for(3000)); + assert_eq!( + l.reclaim_delay(2, 1024, 0, s), + None, + "armed with nothing grown" + ); + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + assert_eq!( + l.reclaim_delay(2, 32, 0, s), + None, + "armed with nothing grown" + ); + } + + /// A client-boundary re-initialization with 23 chunks backed and no render after it: the + /// kept backing is due to be freed after the long idle, measured from the start of the idle, + /// and once freed nothing more is due. + #[test] + fn pending_reinit_frees_kept_backing() { + let s = 1_000_000_000; + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 2800; + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + // The poll after the 50 ms re-initialization waits for the rest of the long idle. + assert_eq!( + l.pending_free_delay(2, DRAIN_IDLE_NS, s), + Some(s - DRAIN_IDLE_NS) + ); + assert!(!l.pending_free_due(s - 1, s)); + assert!(l.pending_free_due(s, s)); + assert_eq!(l.pending_free_delay(2, s + 1, s), Some(0)); + assert_eq!(l.pending_free_delay(1, 0, s), None); + assert_eq!(l.pending_free_delay(0, 0, s), None); + l.unback_to(l.keep_chunks()); + assert!(!l.pending_free_due(100 * s, s)); + assert_eq!(l.pending_free_delay(2, 0, s), None); + } + + #[test] + fn grants_only_backed_blocks() { + let mut l = Ledger::new(true, 64, 4095); + // Nothing backed above the initial posting: a request waits and keeps its counter. + l.backed = 64; + let q = request(3, 0, 1); + assert_eq!(l.accept(&q), Answer::Wait); + assert_eq!(l.next_request, 0); + // A grant is clipped to the backed blocks; the next request waits again. + l.backed = 70; + assert_eq!( + l.accept(&q), + Answer::Grant { + first: 64, + count: 6 + } + ); + l.post(6); + assert_eq!(l.next_request, 1); + assert_eq!(l.accept(&request(3, 1, 1)), Answer::Wait); + // At the maximum the refusal does not wait for backing. + l.posted = l.max; + l.backed = l.max; + assert_eq!(l.accept(&request(3, 1, 1)), Answer::Refuse); + // A re-initialization keeps the backing. + l.reinit(); + assert_eq!((l.posted, l.backed, l.next_request), (64, 4095, 0)); } #[test] diff --git a/drivers/gpu/drm/asahi/g15/recovery.rs b/drivers/gpu/drm/asahi/g15/recovery.rs index b81734b1d5f2d..e03b0d6661f93 100644 --- a/drivers/gpu/drm/asahi/g15/recovery.rs +++ b/drivers/gpu/drm/asahi/g15/recovery.rs @@ -224,7 +224,7 @@ impl Firmware { for flight in self.engine.flights.iter_mut().skip(1) { flight.doomed = true; } - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); let kind = if offset == logic::FRAGMENT_KILL { "fragment" } else { @@ -281,7 +281,7 @@ impl Firmware { if decision.compute_hang { self.report("compute hang"); } - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); if decision.class0_hang { dev_warn!( self.dev(), @@ -323,7 +323,7 @@ impl Firmware { /// anything is published; if it cannot be, the GPU stops. Anything else fail-stops. fn fail_or_recover(&mut self, recover: bool, fault: bool, done: &mut Completions) -> Result { let flight = !self.engine.flights.is_empty(); - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); self.report(if flight { "prefix FAILED" } else { diff --git a/drivers/gpu/drm/asahi/g15/recovery/logic.rs b/drivers/gpu/drm/asahi/g15/recovery/logic.rs index 6d36d1d6550d5..767fb7c952937 100644 --- a/drivers/gpu/drm/asahi/g15/recovery/logic.rs +++ b/drivers/gpu/drm/asahi/g15/recovery/logic.rs @@ -37,9 +37,18 @@ pub(crate) const DRAIN_POLLS: u32 = 1000; pub(crate) const FURTHER_HALTS: u32 = 4; /// Stamp cell `obs` reached the own target `own`, bounded by the newest published target -/// `tail`. Stamps are seeded so that a written stamp is never 0 and has a zero low byte. +/// `tail`. Written stamps have a zero low byte, including at counter wrap. pub(crate) const fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { - obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) + obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) +} + +/// The position of unfinished work; zero is a valid position after stamp wrap. +pub(crate) const fn pending_stamp(obs: u32, own: u32, tail: u32) -> Option { + if stamp_reached(obs, own, tail) { + None + } else { + Some(obs) + } } /// Cursor `c` of a ring of `m` entries passed the end `h` of entries starting at `b`, within @@ -73,9 +82,9 @@ pub(crate) struct Receipts { pub(crate) fs_start: u32, /// Host channel producers (TA, FS, compute) after the prefix's records. pub(crate) rec: [u32; 3], - /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none). + /// Compute member j's FW stamp target, and the FS target it waits on directly. pub(crate) cstamp: [u32; 16], - pub(crate) cwait: [u32; 16], + pub(crate) cwait: [Option; 16], } /// The newest published receipts. @@ -136,8 +145,10 @@ pub(crate) fn compute_hang(f: &Receipts, t: &Tail, o: &Observed) -> bool { if j == computes { return false; } - if f.cwait[j] != 0 && !stamp_reached(o.fs_stamp, f.cwait[j], t.fs_target) { - return false; // still waiting on a render of the prefix + if let Some(target) = f.cwait[j] { + if !stamp_reached(o.fs_stamp, target, t.fs_target) { + return false; // still waiting on a render of the prefix + } } if o.consumer[COMPUTE] != f.rec[COMPUTE] { return false; // its record not consumed yet @@ -332,10 +343,20 @@ mod tests { assert!(!stamp_reached(stamp(4), stamp(1), stamp(3))); // Across the u32 wrap. let own = 0xffffff00u32; + assert!(stamp_reached(0, own, 0x00000200)); + assert!(stamp_reached(0, 0, 0)); + assert!(!stamp_reached(0xffffff00, 0, 0x100)); assert!(stamp_reached(0x00000100, own, 0x00000200)); assert!(!stamp_reached(0xfffffe00, own, 0x00000200)); } + #[test] + fn progress_at_zero_wrap() { + assert_eq!(pending_stamp(0, 0x100, 0x200), Some(0)); + assert_eq!(pending_stamp(0, 0, 0x100), None); + assert_eq!(pending_stamp(0xffffff00, 0, 0x100), Some(0xffffff00)); + } + #[test] fn rings() { // Entries 10..14 of a 0x500 ring, tail at 20. @@ -401,13 +422,19 @@ mod tests { assert!(!compute_hang(&f, &t, &o)); o.compute_read = 15; // The running compute waits on an FS of the prefix that has not completed. - f.cwait[1] = 0x3d000300; + f.cwait[1] = Some(0x3d000300); o.fs_stamp = 0x3d000200; let mut t2 = t; t2.fs_target = 0x3d000400; assert!(!compute_hang(&f, &t2, &o)); o.fs_stamp = 0x3d000300; assert!(compute_hang(&f, &t2, &o)); + f.cwait[1] = Some(0); + t2.fs_target = 0x100; + o.fs_stamp = 0xffffff00; + assert!(!compute_hang(&f, &t2, &o)); + o.fs_stamp = 0; + assert!(compute_hang(&f, &t2, &o)); // No compute in the prefix. f.computes = 0; assert!(!compute_hang(&f, &t, &o)); diff --git a/drivers/gpu/drm/asahi/g15/retain.rs b/drivers/gpu/drm/asahi/g15/retain.rs new file mode 100644 index 0000000000000..30fd79d510869 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/retain.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The once-per-boot module pin taken before the firmware is first touched. +//! +//! A started G15 firmware session cannot be stopped, so this module must stay loaded until +//! reboot from the first firmware-visible write on. `__module_get` alone does not ensure that: +//! a deferred or asynchronous probe can run while an ordinary rmmod is already past its +//! reference check, and the module is then unloaded under a running firmware. +//! `try_module_get` fails once removal has begun, so the pin is taken with it, before the +//! session is marked retained and before anything firmware-visible happens. + +use core::sync::atomic::{AtomicBool, Ordering}; + +/// The module is being removed: nothing was pinned or marked, and nothing may be started. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Removing; + +/// Pins the module once for the session `retained` tracks. `get` is +/// `try_module_get(THIS_MODULE)`, `put` is `module_put(THIS_MODULE)`. +/// +/// Already retained: nothing more is taken. Otherwise the reference comes first; `retained` is +/// set only once it is held. A caller that loses the race to set it drops only its own +/// reference, so exactly one reference stays and it is never put. +pub(crate) fn retain( + retained: &AtomicBool, + get: impl FnOnce() -> bool, + put: impl FnOnce(), +) -> Result<(), Removing> { + if retained.load(Ordering::Acquire) { + return Ok(()); + } + if !get() { + return Err(Removing); + } + if retained.swap(true, Ordering::AcqRel) { + put(); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use core::sync::atomic::AtomicI32; + use std::sync::Arc; + + #[test] + fn removal_fails_closed_with_nothing_pinned_or_marked() { + let retained = AtomicBool::new(false); + let refs = AtomicI32::new(1); // base reference only: rmmod has released it + let put = || panic!("nothing to put"); + assert_eq!(retain(&retained, || false, put), Err(Removing)); + assert!(!retained.load(Ordering::Acquire), "a refused pin must not mark the session"); + assert_eq!(refs.load(Ordering::Acquire), 1); + // A later live start can still pin: the refusal left no state behind. + let get = || { + refs.fetch_add(1, Ordering::AcqRel); + true + }; + assert_eq!(retain(&retained, get, || panic!("winner keeps its reference")), Ok(())); + assert!(retained.load(Ordering::Acquire)); + assert_eq!(refs.load(Ordering::Acquire), 2); + } + + #[test] + fn first_pin_is_kept_and_later_calls_take_nothing() { + let retained = AtomicBool::new(false); + let refs = AtomicI32::new(1); + let get = || { + refs.fetch_add(1, Ordering::AcqRel); + true + }; + assert_eq!(retain(&retained, get, || panic!("winner keeps its reference")), Ok(())); + // The post-start call in Gpu::retain and any later one: no new reference, no put. + for _ in 0..3 { + assert_eq!( + retain(&retained, || panic!("already retained"), || panic!("nothing to put")), + Ok(()) + ); + } + assert_eq!(refs.load(Ordering::Acquire), 2); + } + + #[test] + fn a_lost_race_drops_only_its_own_reference() { + let retained = AtomicBool::new(false); + let refs = AtomicI32::new(2); // base + the reference the winner already holds + let puts = AtomicI32::new(0); + // The winner marks the session between this caller's check and its swap. + let get = || { + refs.fetch_add(1, Ordering::AcqRel); + retained.store(true, Ordering::Release); + true + }; + let put = || { + refs.fetch_sub(1, Ordering::AcqRel); + puts.fetch_add(1, Ordering::AcqRel); + }; + assert_eq!(retain(&retained, get, put), Ok(())); + assert_eq!(puts.load(Ordering::Acquire), 1); + assert_eq!(refs.load(Ordering::Acquire), 2); + } + + #[test] + fn concurrent_pins_leave_exactly_one_reference() { + for _ in 0..500 { + let retained = Arc::new(AtomicBool::new(false)); + let refs = Arc::new(AtomicI32::new(1)); + let start = Arc::new(std::sync::Barrier::new(8)); + let threads: Vec<_> = (0..8) + .map(|_| { + let (retained, refs, start) = (retained.clone(), refs.clone(), start.clone()); + std::thread::spawn(move || { + let (a, b) = (refs.clone(), refs.clone()); + start.wait(); + retain( + &retained, + move || { + a.fetch_add(1, Ordering::AcqRel); + true + }, + move || { + b.fetch_sub(1, Ordering::AcqRel); + }, + ) + }) + }) + .collect(); + for t in threads { + assert_eq!(t.join().unwrap(), Ok(())); + } + assert!(retained.load(Ordering::Acquire)); + assert_eq!(refs.load(Ordering::Acquire), 2, "exactly one kept reference"); + } + } +} diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index d6971489ee80f..53e99f5a772b2 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,8 +25,10 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` -//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) +//! and return them after successful retirement. Failed published jobs retain their slots until +//! their last reference drops (until reboot for jobs retained after a fail-stop). +//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index ccb4f77921e0c..24a912d5968bb 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -10,18 +10,15 @@ //! ([`object::Objects::clients`]) as driver mappings at their fixed GVAs, and reserve //! [`super::hw::client::CLIENT_APERTURE`], the pass-slot range and the PB page list //! (`vm.install_driver_mappings`): user binds there fail (EINVAL). The aperture holds the -//! whole PB range, whose backing is mapped later, before the VM's first prefix is published -//! (`Firmware::map_pb`, counted in [`VmState::pb_chunks`]); -//! - bind it to an exclusive TTBAT slot ([`mmu::Uat::bind`]) for as long as its [`VmState`] -//! lives; at most [`super::hw::sizes::MAX_VMS`] such VMs (ENOSPC beyond, checked before -//! binding so the slot allocator never blocks). +//! whole PB range, whose backing chunks are mapped later, before the VM's prefixes are +//! published (`Firmware::map_pb`) and, for the VM of the flights, when a chunk is backed +//! ([`VmState::map_heap`], [`VmState::map_chunk`]); +//! - leave it unbound until a runnable job needs a TTBAT slot. Each published job holds +//! its binding through completion; failed published jobs retain theirs until their last +//! reference drops (until reboot for jobs retained after a fail-stop). //! -//! The [`VmState`] of a VM is shared by the [`Registry`] (while the client's VM handle exists), -//! the VM's queues and its jobs. It cannot be owned by the [`mmu::Vm`] itself: the slot binding -//! holds a `Vm` reference. The registry drops its reference once the shared lifetime reports the -//! client's handle closed ([`mmu::Vm::is_closed`]), at the next VM creation; the slot returns when -//! the last queue or job of the VM is gone (failed jobs are retained until reboot, and their -//! VM's slot with them). +//! The registry keeps a VM's state while its client handle is open. Queues and jobs retain +//! closed VMs until their last reference drops. //! //! User page tables are cleaned to the point of coherency by the shared `pgtable.rs` (enabled by //! the adopted UAT); user objects are prepared at creation (`gpu::Gpu::prepare_new_objects`). @@ -29,7 +26,7 @@ //! alike. use core::ops::Range; -use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; +use core::sync::atomic::Ordering; use kernel::{ drm::gem::BaseObject, new_mutex, @@ -38,7 +35,7 @@ use kernel::{ }; use super::{hw, object, Gpu}; -use crate::{debug::*, g17::status::VmStatus, gem, mmu, module_parameters}; +use crate::{debug::*, g17::status::VmStatus, gem, mmu}; const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; @@ -104,64 +101,107 @@ impl ClientAlias { } } +/// A PB heap chunk mapped in a VM at its fixed GVA ([`super::pools::chunk_gva`]). +struct HeapMapping { + gem: ARef, + _mapping: mmu::KernelMapping, +} + +/// The PB heap chunks mapped in a VM, in heap order (chunk `i` at `chunk_gva(i)`). Changed +/// under the device lock (refill, publication). +#[pin_data] +struct HeapMappings { + #[pin] + list: Mutex>, +} + /// G15 state of one user VM, shared by the registry, its queues and its jobs. pub(crate) struct VmState { /// The `mmu::Vm` identifier (`Gpu::ids.vm`). pub(crate) id: u64, - /// The VM's TTBAT slot (ASID), for the work images' context. - pub(crate) slot: u32, /// The sticky status (shared with `file.rs`). pub(crate) status: Arc, /// The VM: span checks, stream reads and job guards. pub(crate) vm: mmu::Vm, - /// The exclusive slot binding, released before the live count drops. - bind: Option, + /// The adopted UAT, used to bind published jobs without waiting for a slot. + uat: Arc, registry: Arc, - /// PB backing chunks mapped into the VM, from the first (`Firmware::map_pb`); changed only - /// under the device lock. - pub(crate) pb_chunks: AtomicUsize, + /// The PB heap chunks mapped in this VM. + heap: Arc, } impl VmState { + pub(crate) fn try_bind(&self) -> Result { + self.uat.try_bind(&self.vm) + } + /// Drops the registry's references to closed VMs (the queues call this when destroyed). pub(crate) fn prune_registry(&self) { self.registry.prune(); } -} -impl Drop for VmState { - fn drop(&mut self) { - mod_pr_debug!("G15: VM {} releases context {}\n", self.id, self.slot); - drop(self.bind.take()); - self.registry.live.fetch_sub(1, Ordering::AcqRel); + /// Maps PB heap chunk `i`, the next one this VM lacks, at its GVA. The alias is held only + /// once it is in the list. + fn map_one(&self, heap: &mut KVec, i: usize, gem: ARef) -> Result { + if i != heap.len() { + return Err(EINVAL); + } + heap.reserve(1, GFP_KERNEL)?; + let mapping = self.vm.map_at( + super::pools::chunk_gva(i), + super::pools::CHUNK_BYTES, + &gem, + mmu::Prot::from_pte(hw::pte::LEAF_CACHED), + false, + )?; + heap.push_within_capacity(HeapMapping { + gem, + _mapping: mapping, + }) + .map_err(|_| ENOMEM) } -} -/// Live-VM accounting and the VMs whose client handle is open. -#[pin_data] -pub(crate) struct Registry { - /// VMs whose client handle may still be open, oldest first. - #[pin] - vms: Mutex>>, - /// Live [`VmState`]s (each holds a TTBAT slot) plus reservations being created. - live: AtomicU32, -} + /// Maps the backed chunks (`gems`, in heap order) this VM does not hold yet. A chunk whose + /// object changed since (freed and backed again) is unmapped first, with everything above + /// it, and so are chunks past the backing. + pub(crate) fn map_heap(&self, gems: impl Iterator>) -> Result { + let mut heap = self.heap.list.lock(); + let mut n = 0; + for (i, gem) in gems.enumerate() { + n = i + 1; + if let Some(have) = heap.get(i) { + if core::ptr::eq(&*have.gem, &*gem) { + continue; + } + heap.truncate(i); + } + self.map_one(&mut heap, i, gem)?; + } + heap.truncate(n); + Ok(()) + } -/// A reserved place among the live VMs, released unless a [`VmState`] takes it over. -struct Reservation<'a>(Option<&'a Registry>); + /// Maps the newly backed chunk `i` into this VM (the VM of a flight, at a refill). + pub(crate) fn map_chunk(&self, i: usize, gem: ARef) -> Result { + let mut heap = self.heap.list.lock(); + if i < heap.len() { + // Already mapped (a flight of the same VM came first). + return Ok(()); + } + self.map_one(&mut heap, i, gem) + } -impl Reservation<'_> { - fn into_state(mut self) { - self.0 = None; + /// Unmaps the heap chunks from index `keep` on. + pub(crate) fn unmap_heap_above(&self, keep: usize) { + self.heap.list.lock().truncate(keep); } } -impl Drop for Reservation<'_> { - fn drop(&mut self) { - if let Some(registry) = self.0 { - registry.live.fetch_sub(1, Ordering::AcqRel); - } - } +/// The VMs whose client handle is open. +#[pin_data] +pub(crate) struct Registry { + #[pin] + vms: Mutex>>, } impl Registry { @@ -169,32 +209,14 @@ impl Registry { Arc::pin_init( pin_init!(Self { vms <- new_mutex!(KVec::new(), "g15::Registry::vms"), - live: AtomicU32::new(0), }), GFP_KERNEL, ) } - /// Reserves a place for a new VM after dropping the registry's references to closed VMs. - fn reserve(&self) -> Result> { - self.prune(); - // The shared slot allocator hands out a single slot with robust isolation. - let limit = if module_parameters::robust_isolation.value() != 0 { - 1 - } else { - hw::sizes::MAX_VMS - }; - self.live - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |live| { - (live < limit).then_some(live + 1) - }) - .map_err(|_| ENOSPC)?; - Ok(Reservation(Some(self))) - } - /// Drops the registry's references to VMs whose client handle is closed. Called on VM close /// ([`closed`]), on queue destruction and before a new VM is created: a closed VM's page - /// tables, bound objects and TTBAT slot go as soon as its last queue or job is gone. + /// tables and bound objects go as soon as its last queue or job is gone. fn prune(&self) { let mut closed = KVec::new(); { @@ -251,6 +273,17 @@ impl Registry { .ok_or(ENOENT) } + /// The VMs whose client handle is open (and the closed ones not pruned yet), for a walk + /// that takes VM locks: not under the registry lock. + pub(crate) fn open(&self) -> Result>> { + let vms = self.vms.lock(); + let mut out = KVec::with_capacity(vms.len(), GFP_KERNEL)?; + for state in vms.iter() { + out.push(state.clone(), GFP_KERNEL)?; + } + Ok(out) + } + /// Latches `error` on every VM whose client handle is open. The fail-stop path calls it under /// the device lock. pub(crate) fn fail_all(&self, error: Error) { @@ -286,31 +319,32 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { ClientAlias::collect(&firmware.objects)?, ) }; - let reservation = registry.reserve()?; + registry.prune(); let id = gpu.ids.vm.next(); let vm = gpu.uat.new_vm(id, kernel_range)?.with_tracked_status()?; let status = vm.status().ok_or(EINVAL)?.clone(); let (mappings, reserved) = Registry::driver_mappings(&vm, aliases)?; vm.install_driver_mappings(mappings, reserved)?; - // The context names the root only now: everything above is already visible to the walker. - let bind = gpu.uat.bind(&vm)?; - let slot = bind.slot(); - // From here on the state (even one that fails to allocate) gives the place back. - reservation.into_state(); + // No PB chunk yet: they are mapped before the VM's first prefix (`Firmware::map_pb`). + let heap = Arc::pin_init( + pin_init!(HeapMappings { + list <- new_mutex!(KVec::new(), "g15::HeapMappings::list"), + }), + GFP_KERNEL, + )?; let state = Arc::new( VmState { id, - slot, status, vm: vm.clone(), - bind: Some(bind), + uat: gpu.uat.clone(), registry: registry.clone(), - pb_chunks: AtomicUsize::new(0), + heap, }, GFP_KERNEL, )?; registry.vms.lock().push(state, GFP_KERNEL)?; - mod_pr_debug!("G15: VM {} context {}\n", id, slot); + mod_pr_debug!("G15: VM {} created\n", id); Ok(vm) } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index e9f1008fc5a72..a91f267f9e1e0 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -35,6 +35,7 @@ use kernel::{ }, macros::versions, new_mutex, + new_spinlock, prelude::*, soc::apple::rtkit, sync::{ @@ -44,6 +45,7 @@ use kernel::{ }, Arc, Mutex, + SpinLock, UniqueArc, // }, time::{ @@ -84,6 +86,7 @@ use crate::{ queue, regs, stats, + stats_raw, workqueue, // }; @@ -258,6 +261,9 @@ pub(crate) struct GpuManager { pipes: PipeChannels::ver, event_manager: Arc, stats: Arc, + /// Last raw firmware statistics of this device, exported read-only + /// through `stats_raw`. + raw_stats: Arc>, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, #[pin] @@ -274,6 +280,11 @@ pub(crate) trait Gpu: Send + Sync { /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; + /// The raw firmware statistics snapshot of this GPU, if its firmware reports any. + fn raw_stats_snapshot(&self) -> Option>> { + None + } + /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -827,6 +838,12 @@ impl GpuManager::ver { let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; let stats_snap_clone = stats_snap.clone(); + // Per-device raw statistics snapshot, shared with the debugfs export. + let raw_stats_snap = Arc::pin_init( + new_spinlock!(stats_raw::RawStatsSnapshot::default(), "asahi_stats_raw"), + GFP_KERNEL, + )?; + let raw_stats_snap_clone = raw_stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -842,6 +859,7 @@ impl GpuManager::ver { dev.into(), alloc_ref, stats_snap_clone, + raw_stats_snap_clone, u64::from(cfg.base_clock_hz), )?, }), @@ -869,6 +887,7 @@ impl GpuManager::ver { crashed: AtomicBool::new(false), event_manager, stats: stats_snap.clone(), + raw_stats: raw_stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1324,6 +1343,10 @@ impl Gpu for GpuManager::ver { Some(self.stats.clone()) } + fn raw_stats_snapshot(&self) -> Option>> { + Some(self.raw_stats.clone()) + } + fn as_any(&self) -> &dyn Any { self } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 6baaa97977f5a..a7eb41ccb13b0 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -411,6 +411,21 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +pub(crate) struct VmReadGuard(VmExecGuard); + +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +impl VmReadGuard { + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { + Vm::covers_range_locked(&mut self.0, address, size, read, write) + } + + pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { + Vm::read_user_locked(&mut self.0, va, buf) + } +} + /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, @@ -2191,7 +2206,16 @@ impl Vm { /// mappings cannot change during the copy (GPUVM exec lock). #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - let mut inner = lock_vm(&self.gpuvm, None, false)?; + self.read_lock()?.read_user(va, buf) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + pub(crate) fn read_lock(&self) -> Result { + Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; @@ -3219,6 +3243,16 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, true) + } + + /// Binds a VM without waiting for another slot owner to retire. + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn try_bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, false) + } + + fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { self.inner.check_m4()?; let mut binding = vm.binding.lock(); self.inner.check_m4()?; @@ -3234,7 +3268,11 @@ impl Uat { None }); - let slot = self.slots.get(binding.bind_token)?; + let slot = if wait { + self.slots.get(binding.bind_token)? + } else { + self.slots.try_get(binding.bind_token)? + }; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; diff --git a/drivers/gpu/drm/asahi/slotalloc.rs b/drivers/gpu/drm/asahi/slotalloc.rs index fde7470fe5779..a504e7491bfda 100644 --- a/drivers/gpu/drm/asahi/slotalloc.rs +++ b/drivers/gpu/drm/asahi/slotalloc.rs @@ -216,6 +216,20 @@ impl SlotAllocator { &self, token: Option, cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + ) -> Result> { + self.get_inner_impl(token, cb, true) + } + + /// Gets a slot without waiting; returns ENOSPC when every slot is held. + pub(crate) fn try_get(&self, token: Option) -> Result> { + self.get_inner_impl(token, |_a, _b| Ok(()), false) + } + + fn get_inner_impl( + &self, + token: Option, + cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + wait: bool, ) -> Result> { let mut inner = self.0.inner.lock(); @@ -256,6 +270,9 @@ impl SlotAllocator { } if oldest_time == u64::MAX { + if !wait { + return Err(ENOSPC); + } if first && inner.slot_limit == usize::MAX { pr_warn!( "{}: out of slots, blocking\n", diff --git a/drivers/gpu/drm/asahi/stats_raw.rs b/drivers/gpu/drm/asahi/stats_raw.rs new file mode 100644 index 0000000000000..3d158337f66b7 --- /dev/null +++ b/drivers/gpu/drm/asahi/stats_raw.rs @@ -0,0 +1,501 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Raw firmware statistics snapshot and its read-only debugfs export. +//! +//! The AGX firmware sends statistics messages on the stats channel +//! (`channel::StatsChannel`). They are recorded here exactly as they arrive: +//! every field keeps the value the firmware sent, no unit is assumed, nothing +//! is scaled and no metric is derived from it. `FwBusy` is the firmware's own +//! busy flag and is *not* a GPU utilisation figure; the `Utilization` counters +//! are raw per-subqueue counters whose unit this driver does not assert. +//! +//! This complements the interpreted `stats` snapshot behind `agx_stats`: the +//! snapshot here keeps every message kind the firmware sends, together with +//! per-kind sample counters and capture times, and it never derives a value +//! from the firmware words. +//! +//! Every message kind keeps its own sample counter and the monotonic time at +//! which the sample was captured, so a reader can tell a missing sample +//! (count 0) from a stale one (old capture time), and can compute rates and +//! deltas between two reads. +//! +//! The snapshot is fixed size: recording a sample never allocates. It is +//! guarded by a spin lock because the critical section is short: a sample +//! is stored under the lock and a reader copies the snapshot out, so the lock +//! is never held across formatting. + +use core::fmt::Write; + +use kernel::{ + bindings, + prelude::*, + seq_file::SeqFile, + sync::{ + Arc, + SpinLock, // + }, +}; + +use crate::driver::AsahiDrmDevice; + +/// Format tag of the export. Bumped whenever the field set changes. +pub(crate) const STATS_FORMAT: &str = "asahi_stats_v1"; + +/// The firmware statistics message kinds that are recorded. +/// +/// The discriminants index the per-kind counters of [`RawStatsSnapshot`]; the +/// firmware message tag of each kind is given in its documentation, so every +/// exported line can be mapped back to the message it came from. +#[derive(Copy, Clone, PartialEq, Eq)] +#[repr(usize)] +pub(crate) enum StatsKind { + /// 0x00 `Power`. + Power = 0, + /// 0x02 `PowerOn`. + PowerOn = 1, + /// 0x03 `PowerOff`. + PowerOff = 2, + /// 0x04 `Utilization`. + Utilization = 3, + /// 0x09 `AvgPower`. + AvgPower = 4, + /// 0x0a `Temperature`. + Temperature = 5, + /// 0x0b `PowerState`. + PowerState = 6, + /// 0x0c `FwBusy`. + FwBusy = 7, + /// 0x0d `PState`. + PState = 8, + /// 0x0e `TempSensor`. + TempSensor = 9, +} + +impl StatsKind { + /// Number of recorded kinds. + pub(crate) const COUNT: usize = 10; + + /// All kinds, in export order. + const ALL: [StatsKind; StatsKind::COUNT] = [ + StatsKind::Power, + StatsKind::PowerOn, + StatsKind::PowerOff, + StatsKind::Utilization, + StatsKind::AvgPower, + StatsKind::Temperature, + StatsKind::PowerState, + StatsKind::FwBusy, + StatsKind::PState, + StatsKind::TempSensor, + ]; + + /// Stable name of the kind in the export. + fn name(self) -> &'static str { + match self { + StatsKind::Power => "power", + StatsKind::PowerOn => "power_on", + StatsKind::PowerOff => "power_off", + StatsKind::Utilization => "utilization", + StatsKind::AvgPower => "avg_power", + StatsKind::Temperature => "temperature", + StatsKind::PowerState => "power_state", + StatsKind::FwBusy => "fw_busy", + StatsKind::PState => "pstate", + StatsKind::TempSensor => "temp_sensor", + } + } +} + +/// Last received firmware statistics of one device. +/// +/// All values are the raw firmware words. The driver knows the unit of none +/// of them and derives nothing from them. +#[derive(Copy, Clone, Default)] +pub(crate) struct RawStatsSnapshot { + /// Number of samples received per kind (0 means: never received). + count: [u64; StatsKind::COUNT], + /// Monotonic capture time (ns) of the last sample per kind. + captured_ns: [u64; StatsKind::COUNT], + /// Number of statistics messages received on the channel. + messages: u64, + + /// 0x00 `Power`: raw power word, unit unknown. + power: u64, + /// 0x02 `PowerOn`: raw off-time word, unit unknown. + power_off_time: u64, + /// 0x03 `PowerOff`: raw on-time word, unit unknown. + power_on_time: u64, + /// 0x04 `Utilization`: raw firmware timestamp and four raw counters. + util_timestamp: u64, + /// 0x04 `Utilization`: the four raw counters. + util: [u32; 4], + /// 0x09 `AvgPower`: raw active time word, unit unknown. + avg_power_active_cs: u64, + /// 0x09 `AvgPower`: raw undecoded word. + avg_power_unk2: u32, + /// 0x09 `AvgPower`: raw undecoded word. + avg_power_unk3: u32, + /// 0x09 `AvgPower`: raw undecoded word. + avg_power_unk4: u32, + /// 0x09 `AvgPower`: raw average power word, unit unknown. + avg_power: u32, + /// 0x0a `Temperature`: raw sensor word. + temp_raw_value: u32, + /// 0x0a `Temperature`: raw scale word, travels with the sample. + temp_scale: u32, + /// 0x0a `Temperature`: raw minimum word. + temp_tmin: u32, + /// 0x0a `Temperature`: raw maximum word. + temp_tmax: u32, + /// 0x0b `PowerState`: raw firmware timestamp. + power_state_timestamp: u64, + /// 0x0b `PowerState`: raw firmware timestamp of the last busy period. + power_state_last_busy_ts: u64, + /// 0x0b `PowerState`: raw active word. + power_state_active: u32, + /// 0x0b `PowerState`: raw poweroff word. + power_state_poweroff: u32, + /// 0x0b `PowerState`: raw undecoded word. + power_state_unk1: u32, + /// 0x0b `PowerState`: raw p-state index. + power_state_pstate: u32, + /// 0x0b `PowerState`: raw undecoded word. + power_state_unk2: u32, + /// 0x0b `PowerState`: raw undecoded word. + power_state_unk3: u32, + /// 0x0c `FwBusy`: raw firmware timestamp. + fw_busy_timestamp: u64, + /// 0x0c `FwBusy`: raw firmware busy flag. + fw_busy: u32, + /// 0x0d `PState`: raw lowest p-state bound. + pstate_min: u32, + /// 0x0d `PState`: raw undecoded word. + pstate_unk1: u32, + /// 0x0d `PState`: raw highest p-state bound. + pstate_max: u32, + /// 0x0d `PState`: raw undecoded word. + pstate_unk2: u32, + /// 0x0e `TempSensor`: raw sensor identifier. + sensor_id: u32, + /// 0x0e `TempSensor`: raw sensor word. + sensor_raw_value: u32, + /// 0x0e `TempSensor`: raw scale word, travels with the sample. + sensor_scale: u32, + /// 0x0e `TempSensor`: raw minimum word. + sensor_tmin: u32, + /// 0x0e `TempSensor`: raw maximum word. + sensor_tmax: u32, +} + +impl RawStatsSnapshot { + /// Note one statistics message on the channel. + pub(crate) fn note_message(&mut self) { + self.messages += 1; + } + + /// Record one sample of `kind` captured at `stamp` (monotonic ns). + fn record(&mut self, kind: StatsKind, stamp: u64) { + self.count[kind as usize] += 1; + self.captured_ns[kind as usize] = stamp; + } + + /// Record the 0x00 `Power` sample. + pub(crate) fn set_power(&mut self, power: u64, stamp: u64) { + self.power = power; + self.record(StatsKind::Power, stamp); + } + + /// Record the 0x02 `PowerOn` sample. + pub(crate) fn set_power_on(&mut self, off_time: u64, stamp: u64) { + self.power_off_time = off_time; + self.record(StatsKind::PowerOn, stamp); + } + + /// Record the 0x03 `PowerOff` sample. + pub(crate) fn set_power_off(&mut self, on_time: u64, stamp: u64) { + self.power_on_time = on_time; + self.record(StatsKind::PowerOff, stamp); + } + + /// Record the 0x04 `Utilization` sample. + pub(crate) fn set_utilization(&mut self, timestamp: u64, util: [u32; 4], stamp: u64) { + self.util_timestamp = timestamp; + self.util = util; + self.record(StatsKind::Utilization, stamp); + } + + /// Record the 0x09 `AvgPower` sample. + pub(crate) fn set_avg_power( + &mut self, + active_cs: u64, + unk: [u32; 3], + avg_power: u32, + stamp: u64, + ) { + self.avg_power_active_cs = active_cs; + self.avg_power_unk2 = unk[0]; + self.avg_power_unk3 = unk[1]; + self.avg_power_unk4 = unk[2]; + self.avg_power = avg_power; + self.record(StatsKind::AvgPower, stamp); + } + + /// Record the 0x0a `Temperature` sample. + pub(crate) fn set_temperature( + &mut self, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + stamp: u64, + ) { + self.temp_raw_value = raw_value; + self.temp_scale = scale; + self.temp_tmin = tmin; + self.temp_tmax = tmax; + self.record(StatsKind::Temperature, stamp); + } + + /// Record the 0x0b `PowerState` sample. + #[allow(clippy::too_many_arguments)] + pub(crate) fn set_power_state( + &mut self, + timestamp: u64, + last_busy_ts: u64, + active: u32, + poweroff: u32, + unk1: u32, + pstate: u32, + unk2: u32, + unk3: u32, + stamp: u64, + ) { + self.power_state_timestamp = timestamp; + self.power_state_last_busy_ts = last_busy_ts; + self.power_state_active = active; + self.power_state_poweroff = poweroff; + self.power_state_unk1 = unk1; + self.power_state_pstate = pstate; + self.power_state_unk2 = unk2; + self.power_state_unk3 = unk3; + self.record(StatsKind::PowerState, stamp); + } + + /// Record the 0x0c `FwBusy` sample. + pub(crate) fn set_fw_busy(&mut self, timestamp: u64, busy: u32, stamp: u64) { + self.fw_busy_timestamp = timestamp; + self.fw_busy = busy; + self.record(StatsKind::FwBusy, stamp); + } + + /// Record the 0x0d `PState` sample. + pub(crate) fn set_pstate( + &mut self, + ps_min: u32, + unk1: u32, + ps_max: u32, + unk2: u32, + stamp: u64, + ) { + self.pstate_min = ps_min; + self.pstate_unk1 = unk1; + self.pstate_max = ps_max; + self.pstate_unk2 = unk2; + self.record(StatsKind::PState, stamp); + } + + /// Record the 0x0e `TempSensor` sample. + pub(crate) fn set_temp_sensor( + &mut self, + sensor_id: u32, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + stamp: u64, + ) { + self.sensor_id = sensor_id; + self.sensor_raw_value = raw_value; + self.sensor_scale = scale; + self.sensor_tmin = tmin; + self.sensor_tmax = tmax; + self.record(StatsKind::TempSensor, stamp); + } + + /// Write the raw payload fields of one kind. + fn write_payload(&self, kind: StatsKind, out: &mut impl Write) -> core::fmt::Result { + match kind { + StatsKind::Power => write!(out, " power={}", self.power), + StatsKind::PowerOn => write!(out, " off_time={}", self.power_off_time), + StatsKind::PowerOff => write!(out, " on_time={}", self.power_on_time), + StatsKind::Utilization => write!( + out, + " timestamp={} util1={} util2={} util3={} util4={}", + self.util_timestamp, self.util[0], self.util[1], self.util[2], self.util[3] + ), + StatsKind::AvgPower => write!( + out, + " active_cs={} unk2={} unk3={} unk4={} avg_power={}", + self.avg_power_active_cs, + self.avg_power_unk2, + self.avg_power_unk3, + self.avg_power_unk4, + self.avg_power + ), + StatsKind::Temperature => write!( + out, + " raw_value={} scale={} tmin={} tmax={}", + self.temp_raw_value, self.temp_scale, self.temp_tmin, self.temp_tmax + ), + StatsKind::PowerState => write!( + out, + " timestamp={} last_busy_ts={} active={} poweroff={} unk1={} pstate={} unk2={} unk3={}", + self.power_state_timestamp, + self.power_state_last_busy_ts, + self.power_state_active, + self.power_state_poweroff, + self.power_state_unk1, + self.power_state_pstate, + self.power_state_unk2, + self.power_state_unk3 + ), + StatsKind::FwBusy => { + write!(out, " timestamp={} busy={}", self.fw_busy_timestamp, self.fw_busy) + } + StatsKind::PState => write!( + out, + " ps_min={} unk1={} ps_max={} unk2={}", + self.pstate_min, self.pstate_unk1, self.pstate_max, self.pstate_unk2 + ), + StatsKind::TempSensor => write!( + out, + " sensor_id={} raw_value={} scale={} tmin={} tmax={}", + self.sensor_id, self.sensor_raw_value, self.sensor_scale, self.sensor_tmin, self.sensor_tmax + ), + } + } +} + +/// Write the snapshot in the documented text format. +/// +/// One line per record, `key=value` pairs, LF terminated. A kind that was +/// never received is printed with `count=0` and without payload, so a missing +/// sample is never confused with a zero value. `now_ns` and `captured_ns` are +/// monotonic (CLOCK_MONOTONIC) nanoseconds taken by the driver; every other +/// value is a raw firmware word whose unit this driver does not assert. +pub(crate) fn format_snapshot( + snap: &RawStatsSnapshot, + now_ns: u64, + device: &str, + out: &mut impl Write, +) -> core::fmt::Result { + writeln!(out, "{}", STATS_FORMAT)?; + writeln!(out, "device={}", device)?; + writeln!(out, "now_ns={}", now_ns)?; + writeln!(out, "messages count={}", snap.messages)?; + for kind in StatsKind::ALL { + let count = snap.count[kind as usize]; + write!(out, "{} count={}", kind.name(), count)?; + if count != 0 { + write!(out, " captured_ns={}", snap.captured_ns[kind as usize])?; + snap.write_payload(kind, out)?; + } + writeln!(out)?; + } + Ok(()) +} + +/// Current monotonic time in nanoseconds. +pub(crate) fn monotonic_ns() -> u64 { + // SAFETY: `ktime_get()` may be called from any context. + unsafe { bindings::ktime_get() as u64 } +} + +/// Publish the firmware statistics of one device as a read-only debugfs file. +/// +/// The file is created in the DRM device's own debugfs directory +/// (`/sys/kernel/debug/dri//asahi_stats`), so a card only ever exposes +/// its own statistics. The DRM core removes it when the device is +/// unregistered, and the caller keeps the snapshot alive until after that +/// registration is dropped, so the pointer stays valid for as long as the +/// file exists. +pub(crate) fn register(drm: &AsahiDrmDevice, stats: &Arc>) { + // `drm::Device` is `repr(C)` with the `drm_device` as its first field. + let raw = (drm as *const AsahiDrmDevice) + .cast_mut() + .cast::(); + // `drm_debugfs_add_file()` is a static inline stub without CONFIG_DEBUG_FS, so + // bindgen has no binding for it and there is no file to create. + #[cfg(CONFIG_DEBUG_FS)] + // SAFETY: `raw` is the live DRM device. It is registered at this point, + // so its debugfs directory exists and the file is created in it. + unsafe { + bindings::drm_debugfs_add_file( + raw, + c"asahi_stats".as_char_ptr(), + Some(show), + Arc::as_ptr(stats).cast_mut().cast::(), + ); + } + #[cfg(not(CONFIG_DEBUG_FS))] + let _ = (raw, stats); +} + +/// `drm_debugfs_add_file()` show callback. +/// +/// The DRM core opens the file with `single_open(file, node->show, entry)`, so +/// the seq file's private data is the `struct drm_debugfs_entry` and the +/// per-device payload is in `entry->file.data` (see `drm_debugfs_entry_open()` +/// in `drivers/gpu/drm/drm_debugfs.c`). The `data` argument of the callback is +/// the seq file start token, not the payload. +#[cfg_attr(not(CONFIG_DEBUG_FS), allow(dead_code))] +unsafe extern "C" fn show( + m: *mut bindings::seq_file, + _data: *mut core::ffi::c_void, +) -> core::ffi::c_int { + // SAFETY: The DRM core passes a valid seq file to the show callback. + let seq = unsafe { SeqFile::from_raw(m) }; + // SAFETY: The seq file was opened with the DRM entry as private data. + let entry = unsafe { (*m).private.cast::() }; + // SAFETY: `entry->file.data` is the payload pointer we registered. + let stats = unsafe { (*entry).file.data.cast::>() }; + + let mut out = SeqWriter { seq }; + // SAFETY: The device-owned snapshot outlives its debugfs entry. + let snapshot = *unsafe { (*stats).lock() }; + let device = device_name(entry.cast_const()); + let _ = format_snapshot(&snapshot, monotonic_ns(), device, &mut out); + 0 +} + +/// Name of the DRM device the statistics belong to. +/// +/// The name is owned by the DRM device, which outlives the read. +fn device_name<'a>(entry: *const bindings::drm_debugfs_entry) -> &'a str { + // SAFETY: The DRM core passes the entry it allocated for this file. + let drm = unsafe { (*entry).dev }; + if drm.is_null() { + return "unknown"; + } + // SAFETY: `drm` is the live DRM device of this entry. + let unique = unsafe { (*drm).unique }; + if unique.is_null() { + return "unknown"; + } + // SAFETY: `unique` is a NUL terminated string owned by the DRM device, + // which outlives this read. + let name = unsafe { core::ffi::CStr::from_ptr(unique) }; + name.to_str().unwrap_or("unknown") +} + +/// [`Write`] adapter that forwards to a seq file. +struct SeqWriter<'a> { + seq: &'a SeqFile, +} + +impl Write for SeqWriter<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + self.seq.call_printf(fmt!("{}", value)); + Ok(()) + } +} diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index f09b38aa310ca..ad9c29ae12325 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -1569,6 +1569,62 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, 50000); } +/* Find and check the report before taking the GPU vote; optionally update it. */ +static int apple_pmp_report_t6030_gfx(struct device *supplier, const bool *enable) +{ + struct apple_pmp_report *rep, *found = NULL; + struct device_node *pmp; + const u8 *table; + int len = 0, ret = -ENODEV; + + if (!IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) || + !of_machine_is_compatible("apple,t6030") || + !of_device_is_compatible(supplier->of_node, "apple,t6030-pmp-v2")) + return -ENODEV; + + /* T6030's ordered soc-device ID 5 is the acknowledged AGX request. */ + table = of_get_property(supplier->of_node, "apple,tunable-soc-device", &len); + if (!apple_pmp_devices_valid(table, len, BIT_ULL(4), BIT_ULL(4), true) || + memcmp(table + 4 * PMP_SOC_DEVICE_SIZE + 116, "AGX\0\0\0\0\0", 8)) + return -EINVAL; + + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->offsets != &apple_pmp_offsets_t6030) + continue; + pmp = of_parse_phandle(rep->dev->of_node, "apple,pmp", 0); + if (pmp == supplier->of_node) { + of_node_put(pmp); + if (found) + goto out; + found = rep; + } else { + of_node_put(pmp); + } + } + if (found) { + if (readq(found->base + found->offsets->status) != PMP_REPORT_READY) + ret = -EAGAIN; + else + ret = enable ? apple_pmp_report_set_state(found, 4, *enable, false) : 0; + } +out: + rcu_read_unlock(); + return ret; +} +int apple_pmp_report_check_t6030_gfx(struct device *supplier) +{ + return apple_pmp_report_t6030_gfx(supplier, NULL); +} +EXPORT_SYMBOL_GPL(apple_pmp_report_check_t6030_gfx); + +/* The RTKit AGX vote alone does not enable PMP's GPU bandwidth controllers. */ +int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable) +{ + return apple_pmp_report_t6030_gfx(supplier, &enable); +} +EXPORT_SYMBOL_GPL(apple_pmp_report_set_t6030_gfx); + static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable) { struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index ba8b498786414..7a99bd1be95a7 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -274,9 +274,14 @@ config APPLE_DISPLAY_GATE nomodeset. It also serves T6031 (M3 Max) device trees with apple,t6031-handoff, - in the same way, but only for a boot with - apple_t6031_display.enable=1 on the kernel command line, and only - with the PMP values the boot loader passes. + in the same way and only with the PMP values the boot loader passes. + There the native display is on by default on the J516C, unless + apple_t6031_display.enable=0; the other T6031 boards need + apple_t6031_display.enable=1 on the kernel command line. The T6031 + GPU driver starts its firmware only after this native display is + up, so apple_t6031_display.enable=0, or a native display that does + not come up, also keeps the GPU off for that boot. It is not a + display-only switch. On T6041 (M4 Pro) device trees it enables the internal display nodes only when the boot loader has handed the running display coprocessor @@ -303,10 +308,11 @@ config APPLE_DISPLAY_GATE_KUNIT_TEST default KUNIT_ALL_TESTS help Runs the display gate on test device trees with the T6030, T6031 - and T8122 display nodes, without Apple hardware. On T6031: off by - default, enabled only by apple_t6031_display.enable=1, only with the - boot loader's handoff marker and this Mac's PMP values, and with - nothing changed when a check fails. On T6030: on by default, as + and T8122 display nodes, without Apple hardware. On T6031: on by + default only on the default board (the J516C), otherwise only with + apple_t6031_display.enable=1, off with apple_t6031_display.enable=0, + only with the boot loader's handoff marker and this Mac's PMP values, + and with nothing changed when a check fails. On T6030: on by default, as before, whatever the T6031 option says. On T8122: on by default, and nothing changed when the handoff or the device tree's own PMP topology does not check out. diff --git a/drivers/soc/apple/display-gate-test.c b/drivers/soc/apple/display-gate-test.c index 73b36e5a722b6..c88a338fdc2cd 100644 --- a/drivers/soc/apple/display-gate-test.c +++ b/drivers/soc/apple/display-gate-test.c @@ -7,9 +7,11 @@ * - T6030 (t6030-gate-test.dtso): on by default, as before, and * the T6031 switch does not change it. Run first: the T6031 and T8122 * trees describe a PMP, and the T6030 gate refuses to add a second one. - * - T6031 (t6031-gate-test.dtso): off by default, enabled only by - * apple_t6031_display.enable=1 with the full handoff, and nothing changed - * when a check fails. + * - T6031 (t6031-gate-test.dtso): on by default only on the default board + * (the J516C; the cases point it at the test tree's own root to run that + * path), otherwise only with apple_t6031_display.enable=1, never with + * apple_t6031_display.enable=0; always with the full handoff, and nothing + * changed when a check fails. * - T8122 (t8122-gate-test.dtso): on by default, with the full handoff, * and nothing changed when a check of the handoff or of the reused PMP * topology fails. @@ -143,6 +145,8 @@ struct gate_test { const struct gt_soc *s; struct of_changeset cs; /* this case's own edits to the test tree */ bool requested, t6031_requested; + int t6031_switch; + const char *t6031_board; }; static struct device_node *__init gt_node(struct kunit *test, const char *path) @@ -392,6 +396,8 @@ static int __init gt_init(struct kunit *test, const struct gt_soc *s) /* As the boot left them: the test kernel's command line sets neither. */ t->requested = gate_requested; t->t6031_requested = gate_t6031_requested; + t->t6031_switch = gate_t6031_switch; + t->t6031_board = gate_t6031_board; of_changeset_init(&t->cs); gate_soc = s->soc; gate_pmp_values = NULL; @@ -399,6 +405,7 @@ static int __init gt_init(struct kunit *test, const struct gt_soc *s) gate_pmp_reused = false; gate_requested = true; gate_t6031_requested = false; + gate_t6031_switch = -1; if (gt_live_soc) kunit_skip(test, "the booted device tree has its own /soc"); return 0; @@ -444,6 +451,8 @@ static void __init gt_exit(struct kunit *test) gate_pmp_values = NULL; gate_requested = t->requested; gate_t6031_requested = t->t6031_requested; + gate_t6031_switch = t->t6031_switch; + gate_t6031_board = t->t6031_board; gate_soc = NULL; } @@ -455,29 +464,82 @@ static void __init gate_t6031_param(struct kunit *test) struct gate_test *t = test->priv; KUNIT_EXPECT_FALSE(test, t->t6031_requested); + KUNIT_EXPECT_EQ(test, t->t6031_switch, -1); + KUNIT_EXPECT_STREQ(test, t->t6031_board, "apple,j516c"); KUNIT_EXPECT_TRUE(test, t->requested); KUNIT_EXPECT_EQ(test, gate_setup_t6031("1"), 0); - KUNIT_EXPECT_TRUE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 1); KUNIT_EXPECT_EQ(test, gate_setup_t6031("0"), 0); - KUNIT_EXPECT_FALSE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 0); + /* A rejected value sets nothing: neither an explicit 1 nor a reset to unset. */ KUNIT_EXPECT_NE(test, gate_setup_t6031("maybe"), 0); - KUNIT_EXPECT_FALSE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 0); KUNIT_EXPECT_NE(test, gate_setup_t6031(NULL), 0); - KUNIT_EXPECT_FALSE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 0); + gate_t6031_switch = -1; + KUNIT_EXPECT_NE(test, gate_setup_t6031("maybe"), 0); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, -1); /* The T6030 and T8122 escapes do not touch it, nor it them. */ KUNIT_EXPECT_EQ(test, gate_setup("0"), 0); KUNIT_EXPECT_FALSE(test, gate_requested); - KUNIT_EXPECT_FALSE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, -1); KUNIT_EXPECT_EQ(test, gate_setup_t6031("1"), 0); KUNIT_EXPECT_FALSE(test, gate_requested); KUNIT_EXPECT_EQ(test, gate_setup_t8122("1"), 0); KUNIT_EXPECT_TRUE(test, gate_requested); - KUNIT_EXPECT_TRUE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 1); +} + +/* An explicit 0 or 1 wins; unset, only the default board asks for it. */ +static void __init gate_t6031_resolution(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, gate_t6031_wanted(-1, true)); + KUNIT_EXPECT_FALSE(test, gate_t6031_wanted(-1, false)); + KUNIT_EXPECT_FALSE(test, gate_t6031_wanted(0, true)); + KUNIT_EXPECT_FALSE(test, gate_t6031_wanted(0, false)); + KUNIT_EXPECT_TRUE(test, gate_t6031_wanted(1, true)); + KUNIT_EXPECT_TRUE(test, gate_t6031_wanted(1, false)); +} + +/* Points the default board at the test tree's root, so gate_run() takes that path. */ +static void __init gt_default_board(struct kunit *test) +{ + const char *root = NULL; + + if (of_property_read_string(of_root, "compatible", &root) || !root) + kunit_skip(test, "the test tree's root has no compatible"); + gate_t6031_board = root; +} + +/* On the default board, the full handoff and no switch: enabled. */ +static void __init gate_t6031_default_board_on(struct kunit *test) +{ + gt_default_board(test); + gt_run(test); + gt_expect_enabled(test); +} + +/* apple_t6031_display.enable=0 still wins on the default board. */ +static void __init gate_t6031_default_board_switched_off(struct kunit *test) +{ + gt_default_board(test); + KUNIT_ASSERT_EQ(test, gate_setup_t6031("0"), 0); + gt_run(test); + gt_expect_untouched(test); +} + +/* The default asks for the display; it does not stand in for the handoff. */ +static void __init gate_t6031_default_board_no_marker(struct kunit *test) +{ + gt_default_board(test); + gt_remove(test, gt_t6031.dcp, "apple,t6031-handoff"); + gt_run(test); + gt_expect_untouched(test); } -/* With a complete handoff and no command line switch, nothing changes. */ +/* Off the default board, with a complete handoff and no switch, nothing changes. */ static void __init gate_t6031_off_by_default(struct kunit *test) { gt_run(test); @@ -826,7 +888,11 @@ static void __init gate_t8122_enabled_again(struct kunit *test) /* The case and suite structs outlive init: only the functions are init code. */ static struct kunit_case gate_t6031_cases[] __refdata = { KUNIT_CASE(gate_t6031_param), + KUNIT_CASE(gate_t6031_resolution), KUNIT_CASE(gate_t6031_off_by_default), + KUNIT_CASE(gate_t6031_default_board_on), + KUNIT_CASE(gate_t6031_default_board_switched_off), + KUNIT_CASE(gate_t6031_default_board_no_marker), KUNIT_CASE(gate_t6031_switched_off), KUNIT_CASE(gate_t6031_enabled), KUNIT_CASE(gate_t6031_no_dcp_marker), diff --git a/drivers/soc/apple/display-gate.c b/drivers/soc/apple/display-gate.c index deb84564bc2bc..803ab87be0c1c 100644 --- a/drivers/soc/apple/display-gate.c +++ b/drivers/soc/apple/display-gate.c @@ -53,9 +53,13 @@ * option. * * On T6031 (M3 Max) the gate serves the same handoff, marked - * apple,t6031-handoff, but only when apple_t6031_display.enable=1 is on the - * kernel command line: the native display is off by default there until it - * has been seen to work. The PMP values must come from the boot loader, in + * apple,t6031-handoff. Its default is scoped to the J516C: there the native + * display is on unless apple_t6031_display.enable=0; the other T6031 boards + * need apple_t6031_display.enable=1. The T6031 GPU starts its firmware only + * after this native display is up, so enable=0 (or a display that never + * comes up) keeps the GPU off for that boot too: it is not a display-only + * switch. The default changes no check of the + * handoff or the PMP below. The PMP values must come from the boot loader, in * /chosen/asahi,t6031-pmp; there is no fallback. Its PMP topology is * reused from the device tree, as on T8122. The T6031 display power * states nest the other way round (the DCP's DISP_FE under DISP_CPU under @@ -160,9 +164,21 @@ static const char gate_t6031_pmp_uuid[] __initconst = "07FF4C5B-CB2E-3DE6-95FF-3 */ static bool gate_requested __initdata = true; -/* apple_t6031_display.enable: the T6031 native display is off unless it is 1. */ +/* + * apple_t6031_display.enable: -1 while it is not on the command line, else 0 + * or 1. Unset, the T6031 native display is on only on gate_t6031_board. + */ +static int gate_t6031_switch __initdata = -1; +static const char *gate_t6031_board __initdata = "apple,j516c"; +/* The resolved request, set by gate_run() on a T6031. */ static bool gate_t6031_requested __initdata; +/* Whether the T6031 native display is requested: an explicit 0 or 1 wins. */ +static bool __init gate_t6031_wanted(int sw, bool default_board) +{ + return sw < 0 ? default_board : sw; +} + /* One SoC's internal display handoff, as the gate checks and completes it */ struct gate_soc { const char *machine; /* root compatible */ @@ -259,8 +275,10 @@ static const struct gate_soc gate_t8122 __initconst = { }; /* - * T6031 (M3 Max): the T6030 handoff at the T6031 addresses, off unless - * apple_t6031_display.enable=1. The PMP values must come from the boot + * T6031 (M3 Max): the T6030 handoff at the T6031 addresses, on by default on + * the J516C and off unless apple_t6031_display.enable=1 on the other boards; + * apple_t6031_display.enable=0 keeps it, and with it the GPU, off everywhere. + * The PMP values must come from the boot * loader: there is no built-in fallback. The PMP image is T6031's own; * any other, the T6030 image included, is refused. The report driver checks * the image's SRAM layout before it starts the PMP. The internal display's @@ -368,10 +386,18 @@ static int __init gate_setup_t8122(char *arg) } early_param("apple_t8122_display.enable", gate_setup_t8122); -/* Its own switch: the T6030 and T8122 options do not change it, nor it them. */ +/* + * Its own switch: the T6030 and T8122 options do not change it, nor it them. + * A value kstrtobool() rejects sets nothing, so the board default stays. + */ static int __init gate_setup_t6031(char *arg) { - return kstrtobool(arg, &gate_t6031_requested); + bool on; + int ret = kstrtobool(arg, &on); + + if (!ret) + gate_t6031_switch = on; + return ret; } early_param("apple_t6031_display.enable", gate_setup_t6031); @@ -1993,8 +2019,17 @@ static void __init gate_run(void) if (gate_soc->describe) gate_describe(); + if (gate_soc == &gate_t6031) { + bool board = of_machine_is_compatible(gate_t6031_board); + + gate_t6031_requested = gate_t6031_wanted(gate_t6031_switch, board); + if (gate_t6031_switch < 0 && board) + gate_info("native display on by default on %s; apple_t6031_display.enable=0 keeps the boot framebuffer\n", + gate_t6031_board); + } + if (!*gate_soc->requested) { - if (gate_soc->opt_in) + if (gate_soc->opt_in && gate_t6031_switch < 0) gate_info("native display not requested (%s is not on the command line), display stays on the boot framebuffer\n", gate_soc->opt_in); else diff --git a/drivers/soc/apple/pmp-export.c b/drivers/soc/apple/pmp-export.c index 120c4f32ae2b3..41091755a77c6 100644 --- a/drivers/soc/apple/pmp-export.c +++ b/drivers/soc/apple/pmp-export.c @@ -58,6 +58,7 @@ EXPORT_SYMBOL_GPL(apple_pmp_link_device); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) { + bool report_gfx; int ret; if ((command != 0x0e && command != 0x0f) || device_id > 15 || enabled > 1) @@ -69,6 +70,43 @@ int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) ret = apple_pmp_report_wait_supplier_ready(pmp_device, 5000); if (ret) return ret; - return apple_pmp_send_power_command(pmp_data, command, device_id, enabled); + /* T6030 needs both the RTKit power vote and the PTD bandwidth report. */ + report_gfx = IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) && + command == 0x0f && device_id == 5 && + of_machine_is_compatible("apple,t6030") && + of_device_is_compatible(pmp_device->of_node, "apple,t6030-pmp-v2"); + if (report_gfx) { + ret = apple_pmp_report_check_t6030_gfx(pmp_device); + if (ret) + return ret; + } + if (report_gfx && !enabled) { + ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); + if (ret) { + /* Clearing was written before the ACK timed out. Restore it. */ + apple_pmp_report_set_t6030_gfx(pmp_device, true); + return ret; + } + } + ret = apple_pmp_send_power_command(pmp_data, command, device_id, enabled); + /* Power is unknown after a lost reply; assert the report conservatively. */ + if (report_gfx && !enabled && ret) + apple_pmp_report_set_t6030_gfx(pmp_device, true); + return ret; } EXPORT_SYMBOL_GPL(apple_pmp_set_device_power); + +/* Reassert after startup, including a report left set by an earlier failure. */ +int apple_pmp_t6030_gpu_ready(void) +{ + int ret; + + guard(mutex)(&pmp_lock); + if (!pmp_data) + return -ENODEV; + ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); + if (ret) + return ret; + return apple_pmp_report_set_t6030_gfx(pmp_device, true); +} +EXPORT_SYMBOL_GPL(apple_pmp_t6030_gpu_ready); diff --git a/include/linux/soc/apple/display.h b/include/linux/soc/apple/display.h index 8978d95d40484..eb71ce91e0919 100644 --- a/include/linux/soc/apple/display.h +++ b/include/linux/soc/apple/display.h @@ -6,5 +6,6 @@ struct device; /* Initial native display admission, before starting retained G15 firmware. */ int apple_drm_t6030_link_device(struct device *consumer); +int apple_drm_t6031_link_device(struct device *consumer); #endif diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index 5e819681ba106..dd3373517bf90 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -18,6 +18,9 @@ int apple_pmp_report_ane_available(const struct device_node *np); int apple_pmp_report_ane_perf(const struct device_node *np, struct device *owner, unsigned int level, bool *held); int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms); +/* Caller holds the PMP owner's vote lock and keeps the supplier bound. */ +int apple_pmp_report_check_t6030_gfx(struct device *supplier); +int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable); int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort); @@ -40,6 +43,16 @@ static inline int apple_pmp_report_wait_supplier_ready(struct device *supplier, return -ENODEV; } +static inline int apple_pmp_report_check_t6030_gfx(struct device *supplier) +{ + return -ENODEV; +} + +static inline int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable) +{ + return -ENODEV; +} + static inline int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout) { diff --git a/include/linux/soc/apple/pmp.h b/include/linux/soc/apple/pmp.h index 2317ed46f42de..8b329bf5d4dd5 100644 --- a/include/linux/soc/apple/pmp.h +++ b/include/linux/soc/apple/pmp.h @@ -8,6 +8,8 @@ struct device; #if IS_REACHABLE(CONFIG_APPLE_PMP) int apple_pmp_link_device(struct device *consumer); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled); +/* After the acknowledged AGX vote and successful GPU firmware startup. */ +int apple_pmp_t6030_gpu_ready(void); #else static inline int apple_pmp_link_device(struct device *consumer) { @@ -17,6 +19,10 @@ static inline int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enab { return -ENODEV; } +static inline int apple_pmp_t6030_gpu_ready(void) +{ + return -ENODEV; +} #endif /* Registration is private to the PMP Rust driver and its C bridge. */ int apple_pmp_register(struct device *dev, const void *data); diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 145711b2ccf00..6a8789909bbc8 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include diff --git a/tools/asahi/asahi-stats-raw-tests.py b/tools/asahi/asahi-stats-raw-tests.py new file mode 100755 index 0000000000000..1dc45655eb548 --- /dev/null +++ b/tools/asahi/asahi-stats-raw-tests.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the raw firmware statistics snapshot and its export format. + +The snapshot keeps the firmware words verbatim: no unit is assumed, nothing is +scaled and no metric is derived. This harness pins that contract, because a +consumer parses the exported text. +""" + +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +stats = (ROOT / "drivers/gpu/drm/asahi/stats_raw.rs").read_text() + + +def block(marker): + """Extract the item starting at `marker`, up to its matching brace.""" + start = stats.index(marker) + index = stats.index("{", start) + 1 + depth = 1 + while depth: + depth += (stats[index] == "{") - (stats[index] == "}") + index += 1 + return stats[start:index] + +format_tag = re.search(r"^pub\(crate\) const STATS_FORMAT: &str = \".*\";$", stats, re.MULTILINE) +assert format_tag is not None + + +source = "\n".join([ + "#![allow(dead_code, unused_variables)]", + "use std::fmt::Write;", + format_tag.group(), + block("#[derive(Copy, Clone, PartialEq, Eq)]"), + block("impl StatsKind"), + block("#[derive(Copy, Clone, Default)]"), + block("impl RawStatsSnapshot"), + block("pub(crate) fn format_snapshot"), + r''' +fn render(snap: &RawStatsSnapshot) -> String { + let mut out = String::new(); + format_snapshot(snap, 1000, "testdev", &mut out).unwrap(); + out +} + +const EMPTY: &str = "\ +asahi_stats_v1 +device=testdev +now_ns=1000 +messages count=0 +power count=0 +power_on count=0 +power_off count=0 +utilization count=0 +avg_power count=0 +temperature count=0 +power_state count=0 +fw_busy count=0 +pstate count=0 +temp_sensor count=0 +"; + +#[test] +fn missing_samples_are_explicit() { + let snap = RawStatsSnapshot::default(); + assert_eq!(render(&snap), EMPTY); + // A kind that was never received carries no payload at all, so a missing + // sample can never be read as a zero value. + for line in EMPTY.lines() { + if line.ends_with("count=0") { + assert_eq!(line.split_whitespace().count(), 2, "{}", line); + } + } +} + +#[test] +fn raw_samples_are_preserved_verbatim() { + let mut snap = RawStatsSnapshot::default(); + snap.note_message(); + snap.set_power(0x1234, 11); + snap.note_message(); + snap.set_power_on(0x2222, 12); + snap.note_message(); + snap.set_power_off(0x3333, 13); + snap.note_message(); + snap.set_utilization(0x4444, [1, 2, 3, 4], 14); + snap.note_message(); + snap.set_avg_power(0x5555, [6, 7, 8], 9, 15); + snap.note_message(); + snap.set_temperature(30000, 1000, 0, 100000, 16); + snap.note_message(); + snap.set_power_state(0x6666, 0x7777, 1, 0, 2, 5, 3, 4, 17); + snap.note_message(); + snap.set_fw_busy(0x8888, 1, 18); + snap.note_message(); + snap.set_pstate(0, 1, 7, 2, 19); + snap.note_message(); + snap.set_temp_sensor(2, 30000, 1000, 0, 100000, 20); + + assert_eq!(render(&snap), "\ +asahi_stats_v1 +device=testdev +now_ns=1000 +messages count=10 +power count=1 captured_ns=11 power=4660 +power_on count=1 captured_ns=12 off_time=8738 +power_off count=1 captured_ns=13 on_time=13107 +utilization count=1 captured_ns=14 timestamp=17476 util1=1 util2=2 util3=3 util4=4 +avg_power count=1 captured_ns=15 active_cs=21845 unk2=6 unk3=7 unk4=8 avg_power=9 +temperature count=1 captured_ns=16 raw_value=30000 scale=1000 tmin=0 tmax=100000 +power_state count=1 captured_ns=17 timestamp=26214 last_busy_ts=30583 active=1 poweroff=0 unk1=2 pstate=5 unk2=3 unk3=4 +fw_busy count=1 captured_ns=18 timestamp=34952 busy=1 +pstate count=1 captured_ns=19 ps_min=0 unk1=1 ps_max=7 unk2=2 +temp_sensor count=1 captured_ns=20 sensor_id=2 raw_value=30000 scale=1000 tmin=0 tmax=100000 +"); +} + +#[test] +fn counters_and_capture_times_track_samples() { + let mut snap = RawStatsSnapshot::default(); + snap.set_fw_busy(1, 1, 100); + snap.set_fw_busy(2, 0, 200); + let text = render(&snap); + assert!(text.contains("fw_busy count=2 captured_ns=200 timestamp=2 busy=0\n"), "{}", text); + // The sample count of a kind that was never received stays at zero. + assert!(text.contains("power count=0\n"), "{}", text); + assert!(text.contains("messages count=0\n"), "{}", text); +} + +#[test] +fn no_metric_semantics_are_invented() { + let mut snap = RawStatsSnapshot::default(); + snap.set_fw_busy(1, 1, 100); + snap.set_utilization(1, [10, 20, 30, 40], 100); + let text = render(&snap); + // The firmware busy flag is not a GPU utilisation figure and the raw + // counters keep their own names; no scaled or renamed metric appears. + for forbidden in ["gpu_busy", "util_pct", "utilization_pct", "power_mw", "temp_c", "freq_mhz"] { + assert!(!text.contains(forbidden), "{} in {}", forbidden, text); + } + assert!(text.contains("fw_busy count=1 captured_ns=100 timestamp=1 busy=1\n"), "{}", text); +} +''', +]) + +with tempfile.TemporaryDirectory(prefix="asahi-stats-export-") as tmp: + source_path = Path(tmp) / "test.rs" + binary = Path(tmp) / "test" + source_path.write_text(source) + subprocess.run(["rustc", "--edition=2021", "--test", str(source_path), + "-o", str(binary)], check=True) + subprocess.run([str(binary)], check=True) diff --git a/tools/asahi/display26-tests.py b/tools/asahi/display26-tests.py index 8d2e497fdcf64..2366766042dc2 100644 --- a/tools/asahi/display26-tests.py +++ b/tools/asahi/display26-tests.py @@ -96,6 +96,7 @@ def main(): #include #include #include +#include #include struct device_node { int id; }; struct device { struct device_node *of_node; bool bound; int refs; }; struct device_link { int unused; }; @@ -117,12 +118,17 @@ def main(): static struct device *get_device(struct device*d){d->refs++;return d;} static void put_device(struct device*d){d->refs--;} static struct device_link *device_link_add(struct device*c,struct device*s,unsigned flags){static struct device_link l;assert(flags==1);return link_fail?NULL:&l;} +/* This harness exercises the J613/25G83 bridge, not the T6030 GFX report. */ +static bool of_machine_is_compatible(const char *name){return !strcmp(name,"apple,t8132");} +static bool of_device_is_compatible(const struct device_node*n,const char*name){return false;} ''') write(tmp, 'bridge.c', '''#include #include "drivers/soc/apple/pmp-export.c" static int ready_result,send_result,calls; int apple_pmp_report_wait_supplier_ready(struct device*d,unsigned timeout){assert(d==pmp_device);assert(timeout==5000);return ready_result;} int apple_pmp_send_power_command(const void*data,u8 command,u16 id,u32 enabled){assert(data==pmp_data);calls++;return send_result;} +int apple_pmp_report_check_t6030_gfx(struct device*d){assert(!"unexpected T6030 report on J613");return -EIO;} +int apple_pmp_report_set_t6030_gfx(struct device*d,bool enabled){assert(!"unexpected T6030 report on J613");return -EIO;} int main(void){ struct device_node n={1},wrong={2}; struct device supplier={&n,true,0},consumer={NULL,false,0}; int data; assert(apple_pmp_set_device_power(0xf,8,1)==-ENODEV); diff --git a/tools/asahi/g15-display-order-tests.py b/tools/asahi/g15-display-order-tests.py new file mode 100644 index 0000000000000..a6666efdf2776 --- /dev/null +++ b/tools/asahi/g15-display-order-tests.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the actual G15 native-display start-order bodies on host models. + +The publication slots, the shared link helper and both SoC wrappers are taken +from drivers/gpu/drm/apple/apple_drv.c, the T6031 glue from g15/glue.c and +admit() from g15/boot.rs, unchanged. OF lookups, device references, links and +the admission steps are host models; no device or firmware is touched. Each +mutant must build and fail an intended assertion. +""" +import argparse +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +APPLE = "drivers/gpu/drm/apple/apple_drv.c" +GLUE = "drivers/gpu/drm/asahi/g15/glue.c" +BOOT = "drivers/gpu/drm/asahi/g15/boot.rs" +MODULE = "drivers/gpu/drm/asahi/g15/mod.rs" + + +def item(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def c_source(root): + apple = (root / APPLE).read_text() + glue = (root / GLUE).read_text() + state = [line for line in apple.splitlines() + if line in ("static DEFINE_MUTEX(apple_g15_display_lock);", + "static struct device *apple_t6030_display;", + "static struct device *apple_t6031_display;")] + assert len(state) == 3, f"publication state not found: {state}" + bodies = [item(apple, marker) for marker in ( + "\nstatic struct device **apple_g15_display_slot(struct device *dev)\n{", + "\nstatic int apple_g15_display_publish(struct device *dev)\n{", + "\nstatic void apple_g15_display_withdraw(struct device *dev)\n{", + "\nstatic int apple_g15_link_device(", + "\nint apple_drm_t6030_link_device(struct device *consumer)\n{", + "\nint apple_drm_t6031_link_device(struct device *consumer)\n{", + )] + bodies.append(item(glue, "\nint asahi_g15_display_prepare(struct device *dev)\n{")) + return "\n".join(state) + "\n\n" + "\n\n".join(bodies) + "\n" + + +C_PREFIX = r''' +#include +#include +#include +#include +#include + +#define ENODEV 19 +#define EBUSY 16 +#define EINVAL 22 +#define EPROBE_DEFER 517 +#define DL_FLAG_AUTOREMOVE_CONSUMER 1u +#define EXPORT_SYMBOL_GPL(sym) extern int sym##_exported + +struct device_node { const char *compatible; bool available; int refs; }; +struct device { struct device_node *of_node; int refs; }; +struct device_link { struct device *consumer, *supplier; unsigned int flags; }; +struct mutex { bool held; }; + +static const char *machine_compat[2]; +static struct device_node *nodes[4]; +static int links_made, link_fail, infos; +static struct device_link last_link; + +static int of_machine_is_compatible(const char *compat) +{ + for (int i = 0; i < 2; i++) + if (machine_compat[i] && !strcmp(machine_compat[i], compat)) + return 1; + return 0; +} +static int of_device_is_compatible(const struct device_node *np, const char *compat) +{ + return np && !strcmp(np->compatible, compat); +} +static struct device_node *of_find_compatible_node(struct device_node *from, const char *type, + const char *compat) +{ + (void)from; (void)type; + for (int i = 0; i < 4; i++) + if (nodes[i] && !strcmp(nodes[i]->compatible, compat)) { + nodes[i]->refs++; + return nodes[i]; + } + return NULL; +} +static bool of_device_is_available(const struct device_node *np) { return np && np->available; } +static void of_node_put(struct device_node *np) { if (np) np->refs--; } +static struct device *get_device(struct device *dev) { dev->refs++; return dev; } +static void put_device(struct device *dev) { if (dev) dev->refs--; } +static struct device_link *device_link_add(struct device *consumer, struct device *supplier, + unsigned int flags) +{ + if (link_fail) + return NULL; + links_made++; + last_link = (struct device_link){ consumer, supplier, flags }; + return &last_link; +} +#define dev_info_once(dev, ...) do { static bool once_; (void)(dev); if (!once_) { once_ = true; infos++; } } while (0) + +static void unlock_(struct mutex **m) { if (*m) { assert((*m)->held); (*m)->held = false; } } +static struct mutex *lock_(struct mutex *m) { assert(!m->held); m->held = true; return m; } +#define DEFINE_MUTEX(name) struct mutex name = { false } +#define guard(type) struct mutex *guard_ __attribute__((cleanup(unlock_))) = lock_ +#define scoped_guard(type, m) for (struct mutex *sg_ __attribute__((cleanup(unlock_))) = lock_(m), *done_ = NULL; !done_; done_ = (struct mutex *)1) +''' + +C_MAIN = r''' +static const char *scenario; +#define CHECK(c) do { if (!(c)) { fprintf(stderr, "%s:%d: %s\n", scenario, __LINE__, #c); exit(1); } } while (0) + +static struct device_node gpu31 = { "apple,agx-t6031", true }, gpu30 = { "apple,agx-t6030", true }; +static struct device_node disp31 = { "apple,t6031-display-subsystem", false }; +static struct device_node disp30 = { "apple,t6030-display-subsystem", true }; +static struct device_node other = { "apple,display-subsystem", true }; +static struct device g31 = { &gpu31 }, g30 = { &gpu30 }, none = { NULL }; +static struct device d31 = { &disp31 }, d31b = { &disp31 }, d30 = { &disp30 }, dother = { &other }; + +static void machine(const char *soc, const char *board) +{ + machine_compat[0] = board; + machine_compat[1] = soc; +} +static void clean(void) +{ + CHECK(!apple_g15_display_lock.held); + for (int i = 0; i < 4; i++) + CHECK(!nodes[i] || nodes[i]->refs == 0); +} +static int prepare(struct device *dev) { int ret = asahi_g15_display_prepare(dev); clean(); return ret; } +static int link30(struct device *dev) { int ret = apple_drm_t6030_link_device(dev); clean(); return ret; } +static int link31(struct device *dev) { int ret = apple_drm_t6031_link_device(dev); clean(); return ret; } +static int publish(struct device *dev) { int ret = apple_g15_display_publish(dev); clean(); return ret; } +static void withdraw(struct device *dev) { apple_g15_display_withdraw(dev); clean(); } + +int main(int argc, char **argv) +{ + scenario = argv[1]; + machine("apple,t6031", "apple,j516c"); + nodes[0] = &gpu31; + nodes[1] = &disp31; + nodes[2] = &disp30; + nodes[3] = &other; + if (!strcmp(scenario, "disabled")) { + /* The gate left the display disabled: no start, no success, said once. */ + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1); + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); + } else if (!strcmp(scenario, "absent")) { + nodes[1] = NULL; + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); + } else if (!strcmp(scenario, "pending")) { + /* Enabled but not published: no start, no success, said once. */ + disp31.available = true; + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); + } else if (!strcmp(scenario, "ready")) { + disp31.available = true; + CHECK(publish(&d31) == 0 && d31.refs == 1); + CHECK(prepare(&g31) == 0 && links_made == 1 && infos == 0); + CHECK(last_link.consumer == &g31 && last_link.supplier == &d31 && + last_link.flags == DL_FLAG_AUTOREMOVE_CONSUMER); + link_fail = 1; + CHECK(prepare(&g31) == -EINVAL); + link_fail = 0; + CHECK(publish(&d31) == 0 && d31.refs == 1); + CHECK(publish(&d31b) == -EBUSY && d31b.refs == 0); + withdraw(&d31b); + CHECK(prepare(&g31) == 0); + withdraw(&d31); + CHECK(d31.refs == 0 && prepare(&g31) == -EPROBE_DEFER); + } else if (!strcmp(scenario, "isolation")) { + disp31.available = true; + CHECK(publish(&d30) == 0 && d30.refs == 1); + CHECK(publish(&dother) == 0 && dother.refs == 0); + withdraw(&dother); + CHECK(prepare(&g31) == -EPROBE_DEFER && links_made == 0); + } else if (!strcmp(scenario, "foreign")) { + disp31.available = true; + CHECK(publish(&d31) == 0); + CHECK(prepare(&g30) == -ENODEV && link31(&g30) == -ENODEV); + CHECK(prepare(&none) == -ENODEV && link31(&none) == -ENODEV); + machine("apple,t6030", "apple,j516s"); + CHECK(link31(&g31) == -ENODEV && links_made == 0); + } else if (!strcmp(scenario, "t6030")) { + machine("apple,t6030", "apple,j516s"); + CHECK(link30(&g30) == -EPROBE_DEFER); + disp31.available = true; + CHECK(publish(&d31) == 0 && link30(&g30) == -EPROBE_DEFER); + CHECK(publish(&d30) == 0 && link30(&g30) == 0 && last_link.supplier == &d30); + withdraw(&d30); + CHECK(d30.refs == 0 && link30(&g30) == -EPROBE_DEFER); + CHECK(link30(&g31) == -ENODEV); + } else { + CHECK(!"unknown scenario"); + } + return 0; +} +''' + +C_SCENARIOS = ["disabled", "absent", "pending", "ready", "isolation", "foreign", "t6030"] + +C_MUTANTS = { + "unpublished-succeeds": ("if (!*display)\n\t\treturn -EPROBE_DEFER;", "if (!*display)\n\t\treturn 0;"), + "disabled-display-skipped": (re.compile(r"\t\tdev_info_once\(dev,[^;]*;\n"), "\t\t{ of_node_put(np); return 0; }\n"), + "pending-display-silent": (re.compile(r"\telse\n\t\tdev_info_once\(dev,[^;]*;\n"), ""), + "t6031-reads-t6030-slot": ("&apple_t6031_display);", "&apple_t6030_display);"), + "t6031-published-as-t6030": ("return &apple_t6031_display;", "return &apple_t6030_display;"), + "no-machine-check": ("!of_machine_is_compatible(machine) ||", ""), + "withdraw-keeps-display": ("*slot = NULL;", ";"), + "publish-without-reference": ("*slot = get_device(dev);", "*slot = dev;"), +} + +RS_PREFIX = r''' +#![allow(dead_code, unused_variables, unused_imports, unused_macros, non_snake_case)] +use std::cell::RefCell; +use std::marker::PhantomData; +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +struct Error(i32); +impl Error { fn from_errno(errno: i32) -> Self { Self(errno) } } +type Result = std::result::Result; +const ENODEV: Error = Error(-19); +const EINVAL: Error = Error(-22); +const EIO: Error = Error(-5); +const EPROBE_DEFER: Error = Error(-517); +macro_rules! dev_err { ($($args:tt)*) => {{}}; } +macro_rules! dev_info { ($($args:tt)*) => {{}}; } +macro_rules! dev_warn { ($($args:tt)*) => {{ warned(); }}; } +struct State { board: bool, machine: bool, pmp: i32, display: i32, sleep_hold: bool, asc_fail: bool } +thread_local! { + static CALLS: RefCell> = const { RefCell::new(Vec::new()) }; + static STATE: RefCell = const { RefCell::new(State { board: true, machine: true, pmp: 0, display: 0, sleep_hold: true, asc_fail: false }) }; + static HOLD: RefCell> = const { RefCell::new(None) }; + static WARNINGS: RefCell = const { RefCell::new(0) }; +} +fn warned() { WARNINGS.with(|w| *w.borrow_mut() += 1); } +mod params { + pub fn sleep_hold() -> bool { super::STATE.with(|s| s.borrow().sleep_hold) } +} +fn call(name: &'static str) { CALLS.with(|c| c.borrow_mut().push(name)); } +mod device { + pub struct Device; + impl Device { pub fn as_raw(&self) -> *mut u8 { std::ptr::null_mut() } } +} +mod platform { + pub struct Device(pub super::device::Device, pub C); + impl Device { pub fn as_ref(&self) -> &super::device::Device { &self.0 } } +} +struct Core<'a>(PhantomData<&'a ()>); +struct AsahiDevice; +mod hw { + #[derive(Copy, Clone, PartialEq, Eq)] pub enum Platform { AdtWithPmp, DeviceTree } + #[derive(Copy, Clone, PartialEq, Eq)] pub enum Board { None, Templates, Blob } +} +struct Config { platform: hw::Platform, machine: &'static str, name: &'static str, firmware: (), board: hw::Board } +mod adt { + pub struct GpuAdt; + pub fn read(_: &super::device::Device, _: &super::Config) -> super::Result { super::call("adt"); Ok(GpuAdt) } +} +mod identity { + pub fn firmware(_: &super::device::Device, _: &super::adt::GpuAdt, _: ()) -> super::Result { super::call("identity"); Ok(()) } +} +mod initdata { + pub fn prepare_board(_: &super::device::Device, _: &super::Config, _: &super::adt::GpuAdt) -> super::Result { + super::call("board_images"); Ok(1) + } +} +mod regs { + pub struct Regs; + impl Regs { + pub fn new(_: &super::platform::Device) -> super::Result { super::call("regs"); Ok(Self) } + pub fn asc_control(&self) -> super::Result { + if super::STATE.with(|s| s.borrow().asc_fail) { Err(super::EIO) } else { Ok(0) } + } + } +} +mod glue { + use super::{call, STATE}; + pub fn board() -> bool { call("board"); STATE.with(|s| s.borrow().board) } + pub fn machine(_: &str) -> bool { call("machine"); STATE.with(|s| s.borrow().machine) } + pub unsafe fn asahi_g15_pmp_prepare(_: *mut core::ffi::c_void) -> core::ffi::c_int { call("pmp"); STATE.with(|s| s.borrow().pmp) } + pub unsafe fn asahi_g15_display_prepare(_: *mut core::ffi::c_void) -> core::ffi::c_int { call("display"); STATE.with(|s| s.borrow().display) } + pub fn dma_coherent(_: &super::device::Device) -> bool { call("coherent"); false } +@HOLD_POWER@ +} +struct Admitted { regs: regs::Regs, adt: adt::GpuAdt, board_images: u32 } +''' + +RS_MAIN = r''' +static T6031: Config = Config { platform: hw::Platform::DeviceTree, machine: "apple,j516c", name: "T6031 G15C", firmware: (), board: hw::Board::Blob }; +static T6030: Config = Config { platform: hw::Platform::AdtWithPmp, machine: "apple,t6030", name: "T6030 G15S", firmware: (), board: hw::Board::Templates }; +const TAIL: [&str; 6] = ["adt", "identity", "coherent", "board_images", "regs", "hold"]; +fn run(cfg: &'static Config, machine: bool, pmp: i32, display: i32) -> (Result<()>, Vec<&'static str>) { + CALLS.with(|c| c.borrow_mut().clear()); + STATE.with(|s| *s.borrow_mut() = State { board: true, machine, pmp, display, sleep_hold: true, asc_fail: false }); + HOLD.with(|h| *h.borrow_mut() = None); + WARNINGS.with(|w| *w.borrow_mut() = 0); + let pdev = platform::Device(device::Device, Core(PhantomData)); + let ret = admit(&pdev, &AsahiDevice, cfg).map(|_| ()); + (ret, CALLS.with(|c| c.borrow().clone())) +} +fn with_tail(head: &[&'static str]) -> Vec<&'static str> { head.iter().chain(TAIL.iter()).copied().collect() } +@SLEEP_HOLD_RUN@ +fn main() { + assert_eq!(run(&T6031, true, 0, -517), (Err(EPROBE_DEFER), vec!["machine", "display"]), "T6031 defers before firmware"); + assert_eq!(run(&T6031, true, 0, -19), (Err(ENODEV), vec!["machine", "display"]), "T6031 refusal stops admission"); + assert_eq!(run(&T6031, true, 0, 0), (Ok(()), with_tail(&["machine", "display"])), "T6031 admitted after its display"); + assert_eq!(run(&T6031, false, 0, 0), (Err(ENODEV), vec!["machine"]), "the machine check comes first"); + assert_eq!(run(&T6030, true, 0, -517), (Ok(()), with_tail(&["board", "pmp"])), "T6030 path unchanged"); + assert_eq!(run(&T6030, true, -517, 0), (Err(EPROBE_DEFER), vec!["board", "pmp"]), "T6030 still defers on its PMP link"); +@SLEEP_HOLD_CHECKS@ +} +''' + +HOLD_PLAIN = ' pub fn hold_power(_: &super::device::Device) { call("hold"); }' +HOLD_SLEEP = (' pub fn hold_power(_: &super::device::Device, sleep_hold: bool) {\n' + ' call("hold"); super::HOLD.with(|h| *h.borrow_mut() = Some(sleep_hold));\n }') +SLEEP_HOLD_RUN = r''' +fn run_hold(cfg: &'static Config, sleep_hold: bool, asc_fail: bool) -> (Result<()>, Vec<&'static str>, Option, u32) { + CALLS.with(|c| c.borrow_mut().clear()); + STATE.with(|s| *s.borrow_mut() = State { board: true, machine: true, pmp: 0, display: 0, sleep_hold, asc_fail }); + HOLD.with(|h| *h.borrow_mut() = None); + WARNINGS.with(|w| *w.borrow_mut() = 0); + let pdev = platform::Device(device::Device, Core(PhantomData)); + let ret = admit(&pdev, &AsahiDevice, cfg).map(|_| ()); + (ret, CALLS.with(|c| c.borrow().clone()), HOLD.with(|h| *h.borrow()), WARNINGS.with(|w| *w.borrow())) +} +fn before_hold(head: &[&'static str]) -> Vec<&'static str> { head.iter().chain(TAIL[..TAIL.len() - 1].iter()).copied().collect() } +''' +SLEEP_HOLD_CHECKS = r''' + // The system-sleep hold: T6031 only, and only while g15_sleep_hold is on; off is warned about. + assert_eq!(run_hold(&T6031, true, false), (Ok(()), with_tail(&["machine", "display"]), Some(true), 0), "T6031 holds gfx through sleep"); + assert_eq!(run_hold(&T6031, false, false), (Ok(()), with_tail(&["machine", "display"]), Some(false), 1), "g15_sleep_hold=0: no sleep hold, one warning"); + assert_eq!(run_hold(&T6030, true, false), (Ok(()), with_tail(&["board", "pmp"]), Some(false), 0), "T6030 never holds through sleep"); + // The ASC read comes before the hold: a failure there leaves no hold behind. + assert_eq!(run_hold(&T6031, true, true), (Err(EIO), before_hold(&["machine", "display"]), None, 0), "T6031: ASC failure takes no hold"); + assert_eq!(run_hold(&T6030, true, true), (Err(EIO), before_hold(&["board", "pmp"]), None, 0), "T6030: ASC failure takes no hold"); +''' +SLEEP_HOLD_MUTANTS = { + "sleep-hold-any-platform": ("let sleep_hold = t6031 && params::sleep_hold();", "let sleep_hold = params::sleep_hold();"), + "sleep-hold-ignores-parameter": ("let sleep_hold = t6031 && params::sleep_hold();", "let sleep_hold = t6031;"), + "no-warning-when-off": ("} else if t6031 {", "} else if false {"), + "asc-read-after-hold": (re.compile(r"(\n\s*)let asc = regs\.asc_control\(\)\?;(.*?glue::hold_power\(dev, sleep_hold\);)", re.S), + r"\2\1let asc = regs.asc_control()?;"), +} + +RS_MUTANTS = { + "display-refusal-ignored": ("dev_err!(dev, \"native display not admitted: {:?}\\n\", err);\n }\n return Err(err);", + "dev_err!(dev, \"native display not admitted: {:?}\\n\", err);\n }"), + "display-order-skipped": ("let ret = unsafe { glue::asahi_g15_display_prepare(dev.as_raw().cast()) };", "let ret = 0;"), +} + + +def mutate(source, change): + old, new = change + if isinstance(old, re.Pattern): + mutated, count = old.subn(new, source, count=1) + else: + count = source.count(old) + mutated = source.replace(old, new, 1) + assert count >= 1, f"mutation anchor not found: {old!r}" + return mutated + + +def compile_c(work, source, name): + src = work / f"{name}.c" + exe = work / name + src.write_text(C_PREFIX + source + C_MAIN) + result = subprocess.run(["gcc", "-std=gnu11", "-Wall", "-Werror", "-Wno-unused-function", + "-o", str(exe), str(src)], capture_output=True, text=True) + if result.returncode: + raise SystemExit(f"{name}: does not build:\n{result.stderr}") + return exe + + +def c_failures(exe): + return [s for s in C_SCENARIOS if subprocess.run([str(exe), s], capture_output=True).returncode] + + +def rs_model(admit): + """The prelude and main for this admit(): with the system-sleep hold argument (g15_sleep_hold) or without.""" + sleep = "glue::hold_power(dev, sleep_hold);" in admit + prefix = RS_PREFIX.replace("@HOLD_POWER@", HOLD_SLEEP if sleep else HOLD_PLAIN) + main = RS_MAIN.replace("@SLEEP_HOLD_RUN@", SLEEP_HOLD_RUN if sleep else "").replace("@SLEEP_HOLD_CHECKS@", SLEEP_HOLD_CHECKS if sleep else "") + return sleep, prefix, main + + +def compile_rs(work, rustc, source, name, model): + src = work / f"{name}.rs" + exe = work / name + src.write_text(model[1] + source + model[2]) + result = subprocess.run([rustc, "--edition=2021", "-o", str(exe), str(src)], capture_output=True, text=True) + if result.returncode: + raise SystemExit(f"{name}: does not build with {rustc}:\n{result.stderr}") + return exe + + +def static_checks(root): + apple = (root / APPLE).read_text() + module = (root / MODULE).read_text() + glue = (root / GLUE).read_text() + compat = re.compile(r'of::DeviceId::new\(c_str!\("([^"]+)"\)\)') + start = module.index("kernel::of_device_table!(\n AUTOLOAD_OF_TABLE,") + autoload = module[start:module.index("\n);", start)] + start = module.index("const OF_TABLE:") + matched = module[start:module.index("]);", start)] + assert compat.findall(autoload) == ["apple,agx-t6030", "apple,agx-t6031"], "autoload table" + assert compat.findall(matched) == ["apple,agx-t6030", "apple,agx-t6031"], "match table" + init = item(apple, "\nstatic int apple_drm_init(struct device *dev)\n{") + order = [init.index(s) for s in ("drm_dev_register(", "drm_client_setup_with_fourcc(", "apple_g15_display_publish(dev)")] + assert order == sorted(order), "publication must follow DRM registration and client setup" + uninit = item(apple, "\nstatic void apple_drm_uninit(struct device *dev)\n{") + assert uninit.index("apple_g15_display_withdraw(dev)") < uninit.index("component_unbind_all("), "withdraw before unbind" + assert "apple_t6030_display_publish" not in apple and "apple_t6030_display_withdraw" not in apple + pmp = item(glue, "\nint asahi_g15_pmp_prepare(struct device *dev)\n{") + assert pmp.index("apple_drm_t6030_link_device(dev)") < pmp.index("apple_pmp_link_device(dev)"), "T6030 order" + return 5 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--rustc", action="append", default=[], help="rustc to build the admit() model with") + args = parser.parse_args() + rustcs = args.rustc or ["rustc"] + checks = static_checks(args.root) + source = c_source(args.root) + admit = item((args.root / BOOT).read_text(), "pub(crate) fn admit(") + model = rs_model(admit) + rs_mutants = dict(RS_MUTANTS, **(SLEEP_HOLD_MUTANTS if model[0] else {})) + with tempfile.TemporaryDirectory(prefix="g15-display-order-", dir=os.environ.get("TMPDIR")) as tmp: + work = Path(tmp) + failed = c_failures(compile_c(work, source, "control")) + assert not failed, f"control scenarios failed: {failed}" + for name, change in C_MUTANTS.items(): + failed = c_failures(compile_c(work, mutate(source, change), f"mutant-{name}")) + assert failed, f"C mutant {name} survived" + print(f"mutant {name}: caught by {', '.join(failed)}") + for rustc in rustcs: + exe = compile_rs(work, rustc, admit, "admit", model) + run = subprocess.run([str(exe)], capture_output=True, text=True) + assert run.returncode == 0, f"admit() control failed with {rustc}:\n{run.stderr}" + for name, change in rs_mutants.items(): + exe = compile_rs(work, rustc, mutate(admit, change), f"admit-{name}", model) + run = subprocess.run([str(exe)], capture_output=True, text=True) + assert run.returncode, f"admit() mutant {name} survived with {rustc}" + print(f"mutant {name} ({rustc}): caught") + print(f"PASS: {checks} static checks, {len(C_SCENARIOS)} C scenarios, " + f"{len(C_MUTANTS)} C mutants, admit() on {len(rustcs)} rustc, {len(rs_mutants)} admit() mutants" + + (", system-sleep hold checks" if model[0] else "")) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/asahi/g15-input-error-tests.py b/tools/asahi/g15-input-error-tests.py index 809411963fd17..2dba24aa49204 100644 --- a/tools/asahi/g15-input-error-tests.py +++ b/tools/asahi/g15-input-error-tests.py @@ -2,8 +2,10 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT """Actual G15 job dependency completion and G17 status/policy bodies on host services. -DMA fence lookup, engine submission, queue-fence ordering and kernel mutexes are -models. No kernel scheduling, GPU access or firmware execution is exercised. +DMA fence lookup, engine submission, queue-fence ordering, VM slot allocation and +kernel mutexes are models. Production job binding/completion and prefix admission +bodies run against them. No kernel scheduling, GPU access or firmware execution +is exercised. """ import argparse import hashlib @@ -33,6 +35,7 @@ def extract(text, marker): type Result = std::result::Result; const EIO:Error=Error(-5);const ENODEV:Error=Error(-19);const ETIMEDOUT:Error=Error(-110); const ECANCELED:Error=Error(-125);const EINVAL:Error=Error(-22);const ENOMEM:Error=Error(-12);const ENOENT:Error=Error(-2); +const ENOSPC:Error=Error(-28);const EAGAIN:Error=Error(-11); macro_rules! pr_warn { ($($arg:tt)*)=>{{ let _=format_args!($($arg)*); }} } macro_rules! mod_pr_debug { ($($arg:tt)*)=>{{ let _=format_args!($($arg)*); }} } struct Mutex(StdMutex);impl Mutex{fn lock(&self)->MutexGuard<'_,T>{self.0.lock().unwrap()}} @@ -45,8 +48,17 @@ def extract(text, marker): mod g17 {use super::*;__POLICY__} mod bindings {pub unsafe fn dma_fence_get_status(p:*const i32)->i32{unsafe{*p}}} struct Fence(i32);impl Fence{fn raw(&self)->*const i32{&self.0}} -struct Vm{status:VmStatus} -struct Progress{signaled:AtomicBool} +struct Binding{slot:u32,drops:Arc} +impl Binding{fn slot(&self)->u32{self.slot}} +impl Drop for Binding{fn drop(&mut self){self.drops.fetch_add(1,Ordering::SeqCst);}} +struct Vm{status:VmStatus,refuse_binding:AtomicBool,next_slot:AtomicU32,allocations:AtomicU32,binding_drops:Arc} +impl Vm{fn try_bind(&self)->Result{ + if self.refuse_binding.load(Ordering::SeqCst){return Err(ENOSPC);} + self.allocations.fetch_add(1,Ordering::SeqCst); + Ok(Binding{slot:self.next_slot.load(Ordering::SeqCst),drops:self.binding_drops.clone()}) +}} +struct Progress{signaled:AtomicBool,next:AtomicU32,sent:AtomicU32} +impl Progress{fn next(&self)->u32{self.next.load(Ordering::SeqCst)}fn sent(&self)->u32{self.sent.load(Ordering::SeqCst)}} struct Guard{quarantines:AtomicU32}impl Guard{fn quarantine(&self){self.quarantines.fetch_add(1,Ordering::SeqCst);}} struct JobFence{vm:Arc,signals:AtomicU32,result:AtomicI32,saw_sticky:AtomicI32,saw_report:AtomicI32,saw_mirror:AtomicI32} impl JobFence{ @@ -61,12 +73,25 @@ def extract(text, marker): } struct Shared{crashed:AtomicBool,submits:AtomicU32,submit_result:Result} impl Shared{fn submit(&self,_:Arc)->Result{self.submits.fetch_add(1,Ordering::SeqCst);self.submit_result}} -struct Job{started:AtomicBool,vm:Arc,progress:Progress,_guard:Guard,fence:JobFence} +struct Job{started:AtomicBool,vm:Arc,progress:Progress,_guard:Guard,fence:JobFence,context:AtomicU32,binding:Mutex>} impl Job {__JOB__} +type Completions=Vec; +fn complete(done:&mut Completions,_job:&Arc,result:Result){done.push(result);} +struct Flight{claims:(),group:Vec>} +impl Flight{fn new(_job:Arc)->Self{Self{claims:(),group:vec![]}}} +struct Slots{releases:u32} +impl Slots{fn release(&mut self,_claims:&mut ()){self.releases+=1;}} +struct Engine{slot_wait:bool} +struct HostFirmware{engine:Engine,slots:Slots,publish_result:Result,publishes:u32} +impl HostFirmware{ + fn new(result:Result)->Self{Self{engine:Engine{slot_wait:false},slots:Slots{releases:0},publish_result:result,publishes:0}} + fn publish_prefix_into(&mut self,_job:&Arc,_flight:&mut Flight)->Result{self.publishes+=1;self.publish_result} + __PREFIX__ +} fn make_job(own:Option)->(Arc,Box){ - let mut mirror=Box::new(AtomicI32::new(0));let vm=Arc::new(Vm{status:VmStatus::new((&mut *mirror as *mut AtomicI32).cast())}); + let mut mirror=Box::new(AtomicI32::new(0));let vm=Arc::new(Vm{status:VmStatus::new((&mut *mirror as *mut AtomicI32).cast()),refuse_binding:AtomicBool::new(false),next_slot:AtomicU32::new(7),allocations:AtomicU32::new(0),binding_drops:Arc::new(AtomicU32::new(0))}); if let Some(error)=own{vm.status.record(error);} - let job=Arc::new(Job{started:AtomicBool::new(false),vm:vm.clone(),progress:Progress{signaled:AtomicBool::new(false)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm,signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)}}); + let job=Arc::new(Job{started:AtomicBool::new(false),vm:vm.clone(),progress:Progress{signaled:AtomicBool::new(false),next:AtomicU32::new(0),sent:AtomicU32::new(0)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm,signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)},context:AtomicU32::new(0),binding:Mutex(StdMutex::new(None))}); (job,mirror) } fn shared(crashed:bool,result:Result)->Arc{Arc::new(Shared{crashed:AtomicBool::new(crashed),submits:AtomicU32::new(0),submit_result:result})} @@ -90,7 +115,7 @@ def extract(text, marker): assert_eq!(job.fence.signals.load(Ordering::SeqCst),1,"dependency completion must signal once"); assert_eq!(job.vm.status.get(),0,"duplicate completion must not poison VM"); // A later job on the same VM is admitted despite the reported client error. - let later=Arc::new(Job{started:AtomicBool::new(false),vm:job.vm.clone(),progress:Progress{signaled:AtomicBool::new(false)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm:job.vm.clone(),signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)}}); + let later=Arc::new(Job{started:AtomicBool::new(false),vm:job.vm.clone(),progress:Progress{signaled:AtomicBool::new(false),next:AtomicU32::new(0),sent:AtomicU32::new(0)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm:job.vm.clone(),signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)},context:AtomicU32::new(0),binding:Mutex(StdMutex::new(None))}); later.start(&device,&[Fence(1)]);assert_eq!(device.submits.load(Ordering::SeqCst),1,"successful dependencies after report must submit"); } for own in [EIO,EINVAL]{for input in [ECANCELED,EIO,ENODEV,ETIMEDOUT,EINVAL]{ @@ -111,8 +136,70 @@ def extract(text, marker): } // An engine-owned fault still quarantines before completion. let(job,_mirror)=make_job(None);job.complete(Err(EIO));assert_eq!(job._guard.quarantines.load(Ordering::SeqCst),1);assert_eq!(job.vm.status.get(),EIO.to_errno()); + // Actual bind/release bodies: a job keeps its slot until explicitly released. + { + let(job,_mirror)=make_job(None); + job.vm.refuse_binding.store(true,Ordering::SeqCst); + assert_eq!(job.bind(),Err(ENOSPC));assert!(job.binding.lock().is_none()); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),0); + job.vm.refuse_binding.store(false,Ordering::SeqCst); + job.bind().unwrap();assert_eq!(job.slot(),7,"binding must publish its slot"); + job.vm.next_slot.store(9,Ordering::SeqCst); + job.bind().unwrap();assert_eq!(job.slot(),7,"repeated bind must keep existing binding"); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),1); + job.release_binding();job.release_binding(); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),1,"release_binding must drop exactly once"); + job.bind().unwrap();assert_eq!(job.slot(),9,"released job may acquire a different slot"); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),2); + } + // An engine failure must not recycle an ASID while retained job references live. + for result in [Ok(()),Err(EIO),Err(ETIMEDOUT),Err(ECANCELED)]{ + let(job,_mirror)=make_job(None);job.bind().unwrap();let drops=job.vm.binding_drops.clone(); + job.complete(result); + if result.is_err(){ + assert!(job.binding.lock().is_some(),"failed engine completion must retain binding"); + assert_eq!(drops.load(Ordering::SeqCst),0); + job.complete(Ok(()));assert!(job.binding.lock().is_some(),"duplicate completion must retain failed binding"); + let retained=job.clone();drop(job); + assert_eq!(drops.load(Ordering::SeqCst),0,"a retained reference must keep the binding"); + drop(retained);assert_eq!(drops.load(Ordering::SeqCst),1,"last reference releases failed binding"); + }else{ + assert!(job.binding.lock().is_none(),"successful engine completion releases binding"); + assert_eq!(drops.load(Ordering::SeqCst),1);drop(job);assert_eq!(drops.load(Ordering::SeqCst),1); + } + } + // Run the real prefix admission wrapper; only firmware publication is modeled. + { + let(job,_mirror)=make_job(None);let mut fw=HostFirmware::new(Ok(()));let mut done=vec![]; + job.vm.refuse_binding.store(true,Ordering::SeqCst); + assert_eq!(fw.publish_prefix(&job,&mut done),Err(EAGAIN)); + assert!(fw.engine.slot_wait,"slot exhaustion must request retry"); + assert_eq!(fw.publishes,0);assert!(done.is_empty());assert_eq!(job.vm.status.get(),0); + job.vm.refuse_binding.store(false,Ordering::SeqCst); + assert_eq!(fw.publish_prefix(&job,&mut done),Ok(()));assert_eq!(fw.publishes,1); + } + for published in [false,true]{ + let(job,_mirror)=make_job(None);job.bind().unwrap(); + job.progress.sent.store(u32::from(published),Ordering::SeqCst); + let mut fw=HostFirmware::new(Err(ENOMEM));let mut done=vec![]; + assert_eq!(fw.publish_prefix(&job,&mut done),Err(ENOMEM)); + assert_eq!(fw.slots.releases,1);assert!(done.is_empty()); + if published{ + assert!(job.binding.lock().is_some(),"published prefix must retain binding on append failure"); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),0); + }else{ + assert!(job.binding.lock().is_none(),"unpublished failure must release binding"); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),1); + } + } + { + let(job,_mirror)=make_job(None);let mut fw=HostFirmware::new(Err(EIO));let mut done=vec![]; + assert_eq!(fw.publish_prefix(&job,&mut done),Err(EIO)); + assert!(job.binding.lock().is_some(),"firmware-visible failure must retain binding"); + assert_eq!(fw.slots.releases,0); + } assert_eq!(unrelated.vm.status.get(),0,"unrelated VM untouched");assert_eq!(unrelated.vm.status.reported_error(),0);assert_eq!(other_mirror.load(Ordering::SeqCst),0);assert_eq!(unrelated.fence.signals.load(Ordering::SeqCst),0); - println!("PASS: actual Job::start/completion and VmStatus/policy: healthy/failed VMs, mirror ordering, one-shot completion, successful input, device faults and isolation"); + println!("PASS: production job/prefix/status bodies: input errors, binding reuse/release/retention, exhaustion retry, published-prefix lifetime and isolation"); } ''' @@ -130,17 +217,25 @@ def main(): current = (ROOT / job_path).read_text() status = (ROOT / "drivers/gpu/drm/asahi/g17/status.rs").read_text() policy = (ROOT / "drivers/gpu/drm/asahi/g17/dependency.rs").read_text() + engine = (ROOT / "drivers/gpu/drm/asahi/g15/engine.rs").read_text() methods = [extract(job, " pub(crate) fn complete("), extract(job, " fn finish("), extract(job, " fn start(self: &Arc")] # The old control needs the current helper for the duplicate-completion test; # it still runs its actual unmodified start, which reproduces the lost input error. methods.append(extract(job if "fn finish_dependency(" in job else current, " fn finish_dependency(")) + # Historical removed-fix controls predate publication bindings. Keep their + # actual start/completion bodies and use current binding helpers alongside. + binding_source = job if "fn release_binding(" in job else current + methods.extend(extract(binding_source, marker) for marker in [ + " pub(crate) fn slot(", " pub(crate) fn bind(", + " pub(crate) fn release_binding("]) bodies = "\n".join(methods) code = PRELUDE.replace("__JOB__", bodies).replace("__STATUS__", "\n".join( extract(status, marker) for marker in [" pub(crate) fn get(", " pub(crate) fn record(", " pub(crate) fn reported_error(", " pub(crate) fn report_failure(", " fn unobserved_teardown("])).replace( - "__POLICY__", extract(policy, "pub(crate) fn inherited_error(")) + "__POLICY__", extract(policy, "pub(crate) fn inherited_error(")).replace( + "__PREFIX__", extract(engine, " fn publish_prefix(")) variants = {"control": code} expected = {} if args.mutations: @@ -154,6 +249,11 @@ def main(): "success-refused": ("if let Some(status) = failed {", "if let Some(status) = failed.or(Some(-125)) {", "successful dependencies after report must submit"), "dependency-completes-twice": (" fn finish_dependency(&self, error: Error) {\n if self.progress.signaled.swap(true, Ordering::AcqRel) {\n return;\n }", " fn finish_dependency(&self, error: Error) {", "dependency completion must signal once"), "mirror-not-published": ("unsafe { AtomicI32::from_ptr(mirror.word) }.store(error, Ordering::SeqCst);", "let _ = mirror;", "mirror must precede fence signal"), + "failed-engine-releases-binding": ("if result.is_ok() || !engine {", "if true {", "failed engine completion must retain binding"), + "binding-release-leaks": ("drop(self.binding.lock().take());", "let _ = self;", "release_binding must drop exactly once"), + "bound-job-rebinds": ("if slot.is_none() {", "if true {", "repeated bind must keep existing binding"), + "slot-exhaustion-not-retried": ("self.engine.slot_wait = true;", "self.engine.slot_wait = false;", "slot exhaustion must request retry"), + "published-prefix-releases-binding": ("if job.progress.next() == job.progress.sent() {", "if true {", "published prefix must retain binding on append failure"), } for name, (old, new, message) in changes.items(): if code.count(old) != 1: @@ -162,7 +262,8 @@ def main(): expected[name] = message variants["control-after"] = code receipt = {"scope": __doc__, "ref": args.ref, "sources": {name: hashlib.sha256(text.encode()).hexdigest() - for name, text in [(job_path, job), ("g17/status.rs", status), ("g17/dependency.rs", policy)]}, "variants": {}} + for name, text in [(job_path, job), ("g15/engine.rs", engine), + ("g17/status.rs", status), ("g17/dependency.rs", policy)]}, "variants": {}} for name, source in variants.items(): path = args.output / (name + ".rs"); path.write_text(source) binary = args.output / name diff --git a/tools/asahi/g15-maintained-handoff-tests.py b/tools/asahi/g15-maintained-handoff-tests.py index 4dd2961d3a8ac..fdafd799645cb 100644 --- a/tools/asahi/g15-maintained-handoff-tests.py +++ b/tools/asahi/g15-maintained-handoff-tests.py @@ -137,10 +137,12 @@ def item(s, marker): OWNER_CALLS.set(0); let bytes = fixture(kind); assert!(parse(&dev, &Adt { data: &bytes }).is_err()); assert_eq!(OWNER_CALLS.get(), 0); } - assert_eq!(BUILDS.len(), 2, "maintained B0 firmware row must not require an absent profile symbol"); - assert_eq!(BUILDS[1].machine.unwrap().to_bytes(), b"apple,j516s"); - assert_eq!((BUILDS[1].text_size, BUILDS[1].data_size, BUILDS[1].marker), (0x5c000, 0x114000, 0x58e88)); - assert_eq!(BUILDS[1].text_sha256, [0xda,0x26,0xec,0xca,0xb1,0xf8,0x59,0x8d,0xa8,0x0c,0x85,0x5e,0xca,0x91,0x65,0x20,0x40,0x6d,0x1d,0x06,0x70,0xf1,0xd4,0x70,0xe7,0x9b,0xb9,0xb0,0x22,0x09,0xdb,0x53]); + assert_eq!(BUILDS.len(), 3, "maintained B0 firmware row must not require an absent profile symbol"); + for (row, machine) in [(1, &b"apple,j516s"[..]), (2, &b"apple,j514s"[..])] { + assert_eq!(BUILDS[row].machine.unwrap().to_bytes(), machine); + assert_eq!((BUILDS[row].text_size, BUILDS[row].data_size, BUILDS[row].marker), (0x5c000, 0x114000, 0x58e88)); + assert_eq!(BUILDS[row].text_sha256, [0xda,0x26,0xec,0xca,0xb1,0xf8,0x59,0x8d,0xa8,0x0c,0x85,0x5e,0xca,0x91,0x65,0x20,0x40,0x6d,0x1d,0x06,0x70,0xf1,0xd4,0x70,0xe7,0x9b,0xb9,0xb0,0x22,0x09,0xdb,0x53]); + } for error in [0, -517, -12] { INIT_ERROR.set(error); let shared = Arc::new(Shared); let result = Rtkit::new(&dev, &shared); assert_eq!(result.is_ok(), error == 0); @@ -156,13 +158,16 @@ def main(): p = argparse.ArgumentParser(description=__doc__); p.add_argument('--output', type=Path); p.add_argument('--mutations', action='store_true'); a=p.parse_args() out=(a.output or Path(tempfile.mkdtemp(prefix='g15-maintained-handoff-'))).resolve(); out.mkdir(parents=True, exist_ok=True) adt=(G15/'adt.rs').read_text(); boot=(G15/'boot.rs').read_text(); hw=(G15/'hw.rs').read_text() + record=(G15/'board/record.rs').read_text() helpers='\n'.join(item(adt,m) for m in ["pub(crate) struct GpuAdt", "struct Adt<'a>", 'fn le32(', 'fn le64(', 'fn strncmp_eq(', "impl Adt<'_>", 'fn collect_perf(', 'fn parse(', 'fn admit_regions(']) constants='\n'.join(re.search(r'pub\(crate\) const '+n+r':[^;]+;', hw).group(0) for n in ['PAGE','PA_MASK','TTBS_SIZE','TABLES_SIZE','HANDOFF_SIZE','SHARED_L2_SIZE','UPPER']) + # GpuAdt carries the T6031 sensor rows, sized by the record builder's production constant. + record_constants=re.search(r'pub\(crate\) const SENSOR_ROW_COUNT:[^;]+;', record).group(0) start=hw.index(' firmware: &['); start=hw.index('[',start); end=start+1; depth=1 while depth: depth+=(hw[end]=='[')-(hw[end]==']'); end+=1 firmware='static BUILDS: &[FirmwareBuild] = &'+hw[start:end]+';\n'+item(hw,'pub(crate) struct FirmwareBuild') constructor=item(boot,' fn new(dev: &device::Device, shared: &Arc)') - source=PREFIX+'\nmod hw {\n'+constants+'\n}\n'+helpers+'\n'+firmware+'\nimpl Rtkit {\n'+constructor+'\n}\n'+CASES + source=PREFIX+'\nmod hw {\n'+constants+'\n}\nmod record {\n'+record_constants+'\n}\n'+helpers+'\n'+firmware+'\nimpl Rtkit {\n'+constructor+'\n}\n'+CASES variants={'actual':source} if a.mutations: variants['skip-owner-check']=source.replace('if !glue::reserved_uat(i as u32, r.start, r.end - r.start) {','if false {') @@ -170,7 +175,7 @@ def main(): variants['drop-reserved-ram']=source.replace('Some(dram_base..dram_base + dram_size), true)?;', 'Some(dram_base..dram_base + dram_size), false)?;') variants['require-obsolete-b0-profile']=source.replace(' FirmwareBuild {\n name: "J516S B0', ' #[cfg(CONFIG_SILICONFOREST_NATIVE_GPU)]\n FirmwareBuild {\n name: "J516S B0') variants['named-mailbox']=source.replace('cookie, core::ptr::null(), 0, &RTKIT_OPS', 'cookie, c"mbox".as_ptr(), 0, &RTKIT_OPS') - receipt={'scope':__doc__,'sources':{p:hashlib.sha256(t.encode()).hexdigest() for p,t in [('adt.rs',adt),('boot.rs',boot),('hw.rs',hw)]},'variants':{}} + receipt={'scope':__doc__,'sources':{p:hashlib.sha256(t.encode()).hexdigest() for p,t in [('adt.rs',adt),('boot.rs',boot),('hw.rs',hw),('board/record.rs',record)]},'variants':{}} for name,text in variants.items(): if name!='actual' and text==source: raise SystemExit('NOT_APPLIED '+name) src=out/(name+'.rs');src.write_text(text); binary=out/name diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 12ee4f16a4955..a1f2d67d7a327 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -3,16 +3,18 @@ """Run the G15 PB growth answer and its on-demand backing, actual bodies, against a model. The harness extracts from drivers/gpu/drm/asahi/g15/pools.rs Firmware::answer_growth(), -answer_request(), map_pb(), running_vm() and back_pb(), Pools::soft_max() and backed(), -PbChunk and Pools::new's PB backing statements; pb_max_blocks() from params.rs; the PB sizes, -tuning and PB leaf word from hw.rs; Prot::from_pte() and is_cached_noncoherent() from -pgtable.rs; and the parameter's default from asahi_core.rs. It includes pools/growth.rs, -fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, -the GEM backing allocation, the shared layer's map_at() and append_driver_mappings(), and the -DEVCTL wait, which logs every allocation, alias map, append, unmap, free, block record, -barrier, producer store, reply and log line in order. Allocation, alias map and append -failures can be injected. Source checks: publish_prefix_into() starts with the PB catch-up, -and pools.rs asserts the PB leaf GPU-only at compile time. +answer_request(), map_pb(), refill_backing(), note_idle(), reclaim_check(), free_backing() and +reclaim_delay_ns(), Pools::soft_max(), map_heap(), free_chunks() and back_chunk(), PbChunk, +chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from vm.rs VmState::map_one(), +map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks() from params.rs, whose +pb_blocks(), pb_reclaim(), pb_idle_ns() and pb_reclaim_blocks() must return the fixed hw.rs tuning +values; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and +is_cached_noncoherent() from pgtable.rs; and the g15_pb_max_blocks default from asahi_core.rs. +It includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, the GEM backing allocation, the shared layer's map_at() +and the DEVCTL wait, which logs every allocation, alias map, unmap, free, block record, barrier, +producer store, reply and log line in order. Allocation and alias map failures can be injected. +Source checks: publish_prefix_into() starts with the PB catch-up, and pools.rs asserts the PB leaf +GPU-only at compile time. What it shows: - a poll pass answers only the requests queued when it starts, in order; one the reply's wait @@ -21,22 +23,42 @@ offset, each producer store follows its records and a barrier, each reply follows its producer store and a barrier, and every producer is in 1..=4095, ending at 4095; - at the maximum two refusals are answered; further requests get no reply, count as bad and - warn once until the ledger is made again, as init() does; + warn once until the ledger is reset, as init() does; - the soft maximum is the parameter clamped to 64..=4095 and then to the ledger's initial..=max; growth answers above it are logged at info level, and the one that reaches it exactly is not; - invalid requests (type, slot, stale counter, epoch, no live render) get no reply, and requests rejected so allocate nothing; -- the backing: the initial blocks only when the GPU starts; chunks of at most 4 MiB, allocated - and held by the running VM before any record or producer names their blocks, never past the - maximum, at most growth::max_chunks(); a failed allocation, alias map or append posts and - answers nothing, leaves the request counter and the VM's count alone and drops what is not - held, and the next request is answered; a failed reply keeps everything; -- a VM's count always equals the aliases it holds, contiguous from the heap's start; a VM that - runs after another grew the backing maps it first (at publication, and again before it grows); - a partial catch-up counts only held aliases; -- the one-VM rule: a catch-up with a flight of another VM, or a count past the chunks, and a - grant with flights of two VMs, are refused (EPROTO) before anything is mapped or posted; +- the backing: the chunks of the initial posting and one reserve chunk when the GPU starts; + chunks of 16 MiB, backed after the replies of a pass (never between them) while fewer than a + reserve chunk remain, mapped into the VM of the flights before any record or producer names + their blocks, never past the heap; with the reserve dry a request waits at the head of the + queue, keeps its counter and is answered once a chunk is backed; a chunk that cannot be mapped + into the running VM is dropped again; a pending re-initialization backs nothing while no + render is published, and still backs for a render published before it; a failed reply keeps + everything; +- a VM's aliases are the chunks, contiguous from the heap's start; a VM that runs after another + grew the backing maps it first (at publication); a partial catch-up keeps the aliases made; a + chunk freed and backed again (an idle reclaim) is unmapped and mapped again at publication; +- the one-VM rule: a catch-up with a flight of another VM is refused (EPROTO) before anything + is mapped; +- the idle reclaim, actual reclaim_check(), free_backing() and note_idle() over a clock: a heavy + client that exits with nothing rendered after it is re-initialized after 50 ms with the backing + kept, and the kept backing is freed after the long idle, its posted count kept until the InitBM; + at reclaim level 1 the backing stays; a recovery's pending re-initialization frees nothing; + driven as the idle poll work arms it (reclaim_delay_ns()), the same exit wakes at 50 ms and at + the long idle and arms nothing after; an idle reclaim arms nothing once its re-initialization + is pending; +- nothing grown since the initialization (an initial posting above half the maximum, or a + reclaim floor below it): no re-initialization and nothing armed; +- the actual reclaim keeps the posted count until the next render's InitBM (a model of the + publication: map_pb() first, then for a render prefix with a re-initialization pending the + ledger reset of Pools::init()): the same client rendering again after 60 ms re-posts the + initial blocks and grows again from the kept backing with nothing allocated; after 1.1 s it + grows again over freshly backed chunks, each mapped before it is posted; a compute-only + submission between the reclaim and the InitBM maps, backs and posts nothing; +- a reclaim whose walk of the VMs fails counts nothing freed; the VM's next publication drops + its stale aliases and releases the pages; - the ledger reset of Pools::init() re-posts from the backing without allocating; - the PB leaf decodes as GPU-only, not cached-noncoherent: dropping an alias is a plain unmap; - Ledger::new() is Ledger::new() as of dba2a110de00 (the carried upstream constructor), capped at @@ -47,8 +69,7 @@ refusal or an unanswered request. The model's VMs have no open/closed state: that a closed VM whose jobs still run keeps its aliases comes from the VM lifetime source (Job and VmJobGuard hold the VM), not from these tests. The ledger reset is not a recovery. The shared layer's -mapping and its rollback are modelled, not run. Running only running_vm()'s or only map_pb()'s -one-VM check on a grant is not distinguishable: each covers it. +mapping is modelled, not run. With --mutations, each mutation below must fail its intended test with the expected message, and the control (a driver maximum of the whole 4096-block heap) must pass, held by the ledger's cap. @@ -75,6 +96,7 @@ HW = f"{G15}/hw.rs" CORE = "drivers/gpu/drm/asahi/asahi_core.rs" ENGINE = f"{G15}/engine.rs" +VM = f"{G15}/vm.rs" PGTABLE = "drivers/gpu/drm/asahi/pgtable.rs" OLD_REF = "dba2a110de00743eeecd41edf23e54ede4491d5e" @@ -97,14 +119,16 @@ } }''' -# Pools::new's PB backing, run verbatim by the model. -POOLS_NEW_BACKING = (' // Backing for the initial blocks; growth adds the rest.\n', ' chunks.push(') +# Pools::new's PB backing, run verbatim by the model (up to its log line). +POOLS_NEW_BACKING = (' // Backing for the initial blocks and the reserve; growth adds the rest.\n', + ' dev_info!(\n') # The compile-time proof that a PB alias drop is a plain unmap. PB_LEAF_ASSERT = 'static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent());' -# The two statements of Pools::init() that start a new ledger. -LEDGER_RESET = ['self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS);', - 'self.exhausted_logged = false;'] +# The two statements of Pools::init() that reset the ledger. +LEDGER_RESET = ['self.ledger.reinit();', + 'self.exhausted_logged = false;', + 'self.reclaim_pending = false;'] MUTATIONS = { # Answer until the queue is empty, including requests queued by the replies' waits. @@ -112,9 +136,9 @@ # Leave the last request queued before the pass for later. 'skip-last-queued': (POOLS, 'for _ in 0..queued {', 'for _ in 0..queued.saturating_sub(1) {'), # End the pass at a request that cannot be processed. - 'return-after-request-error': (POOLS, ' self.pools.stats.bad += 1;\n dev_err!(', - ' self.pools.stats.bad += 1;\n return Ok(());\n' - ' dev_err!('), + 'return-after-request-error': (POOLS, ' self.pools.stats.bad += 1;\n dev_err!(', + ' self.pools.stats.bad += 1;\n return Ok(());\n' + ' dev_err!('), # Answer three refusals at the maximum. 'three-refusals': (GROWTH, 'pub(crate) const MAX_REFUSALS: u32 = 2;', 'pub(crate) const MAX_REFUSALS: u32 = 3;'), # Answer refusals for ever. @@ -140,42 +164,115 @@ 'let nb = clamp(blocks, MIN_BLOCKS, MAX_BLOCKS);\n' ' (nb, clamp(max_blocks, nb, MAX_BLOCKS))\n' ' } else {\n (MAX_BLOCKS, MAX_BLOCKS)'), - # Post blocks without backing them first. - 'grant-without-backing': (POOLS, ' self.back_pb(end, ctx)?;\n let (first, n) = self.pools.ledger.accept(request);', - ' let (first, n) = self.pools.ledger.accept(request);'), - # Back the blocks only after the request is accepted. - 'back-after-accept': (POOLS, ' self.back_pb(end, ctx)?;\n let (first, n) = self.pools.ledger.accept(request);', - ' let (first, n) = self.pools.ledger.accept(request);\n self.back_pb(end, ctx)?;'), - # Count a catch-up alias before the VM holds it. - 'catch-up-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n mapped += 1;\n' - ' vm.pb_chunks.store(mapped, Ordering::Relaxed);', - ' mapped += 1;\n vm.pb_chunks.store(mapped, Ordering::Relaxed);\n' - ' vm.vm.append_driver_mappings(aliases)?;'), - # Count a growth chunk's alias before the VM holds it. - 'growth-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.', - ' vm.pb_chunks.store(self.pools.chunks.len() + 1, Ordering::Relaxed);\n' - ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.'), - # Keep a chunk whose alias the VM never held. - 'keep-unappended-chunk': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.\n' - ' self.pools\n .chunks\n .push_within_capacity(chunk)\n' - ' .map_err(|_| EIO)?;', - ' // Within the capacity checked above.\n self.pools\n .chunks\n' - ' .push_within_capacity(chunk)\n .map_err(|_| EIO)?;\n' - ' vm.vm.append_driver_mappings(aliases)?;'), - # Grow without catching the running VM up first. - 'no-growth-catch-up': (POOLS, ' let vm = self.running_vm(ctx)?;\n self.map_pb(&vm)?;\n', - ' let vm = self.running_vm(ctx)?;\n'), + # Grant without the backing: neither wait nor clip. + 'grant-without-backing': [(GROWTH, ' if room != 0 && self.spare() == 0 {\n return Answer::Wait;\n }\n', ''), + (GROWTH, 'count: GROW_STEP.min(room).min(self.spare()),', 'count: GROW_STEP.min(room),')], + # Allocate before the pass's answers. + 'refill-before-answers': [(POOLS, ' self.refill_backing();\n Ok(())', ' Ok(())'), + (POOLS, ' let queued = self.events.grow.len();\n', + ' self.refill_backing();\n let queued = self.events.grow.len();\n')], + # Back a chunk without mapping it into the running VM. + 'refill-maps-no-flight': (POOLS, ' flight.job.vm.map_chunk(i, gem.clone())?;\n', + ' let _ = (flight, i, gem.clone());\n'), + # Drop a request that waits for backing. + 'no-wait-requeue': (POOLS, ' if self.events.grow.insert_within_capacity(0, request).is_err() {\n' + ' self.pools.stats.bad += 1;\n }\n', ''), + # Keep a chunk that the running VM could not map. + 'refill-failure-keeps-chunk': (POOLS, ' self.pools.chunks.truncate(backed);\n', ''), + # Back chunks while a re-initialization is pending. + 'refill-during-reinit': (POOLS, ' if self.recovery.reinit && !render {\n return;\n }\n', ''), + # Skip the refill on any pending re-initialization, also with a live render (a stall). + 'refill-skip-with-live-render': (POOLS, 'if self.recovery.reinit && !render {', 'if self.recovery.reinit {'), + # Refill for any live flight, also compute-only ones that cannot ask for growth. + 'refill-for-any-flight': (POOLS, 'if self.recovery.reinit && !render {', + 'if self.recovery.reinit && self.engine.flights.is_empty() {'), + # Keep a VM's alias of a chunk that was freed and backed again. + 'map-without-identity': (VM, ' if core::ptr::eq(&*have.gem, &*gem) {', ' if true {'), + # Stop the reclaim check on a reclaim's pending re-initialization (the kept backing stays). + 'pending-reinit-stops-free': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + # Free the kept backing at reclaim level 1. + 'pending-free-at-level-1': (POOLS, 'if level >= 2 && ledger.pending_free_due(idle_ns, min_idle) {', + 'if ledger.pending_free_due(idle_ns, min_idle) {'), + # Do not record that the pending re-initialization is a reclaim's. + 'reclaim-pending-not-set': (POOLS, ' self.pools.reclaim_pending = true;\n', ''), + # Free for any pending re-initialization, also a recovery's. + 'free-on-recovery-reinit': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + # Both checks stop on any pending re-initialization: the code before the kept-backing free. + 'old-reinit-guards': [(POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n')], + # Arm nothing while a reclaim's re-initialization is pending. + 'pending-reinit-arms-nothing': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n'), + # Arm the poll for the expired 50 ms instead of the rest of the long idle. + 'pending-delay-from-drain': (GROWTH, 'Some(min_idle_ns.saturating_sub(idle_ns))', + 'Some(DRAIN_IDLE_NS.saturating_sub(idle_ns))'), + # Re-initialize at an idle with nothing grown (posted above the floor only). + 'reclaim-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns', + 'self.posted > floor && idle_ns >= min_idle_ns'), + # Re-initialize at a client boundary with nothing grown (initial above half the maximum). + 'drain-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS', + 'self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS'), + # Arm the poll with nothing grown. + 'delay-nothing-grown': (GROWTH, ' let grown = self.posted > self.initial;\n', ' let grown = true;\n'), + # Reset the posted count at the idle reclaim instead of at the next render's InitBM. + 'reinit-at-idle-reclaim': (POOLS, ' if level >= 2 && idle {\n freed = self.free_backing();\n', + ' if level >= 2 && idle {\n self.pools.ledger.reinit();\n freed = self.free_backing();\n'), + # Reset the posted count at any reclaim instead of at the next render's InitBM. + 'reinit-at-any-reclaim': (POOLS, ' self.pools.reclaim_pending = true;\n', + ' self.pools.reclaim_pending = true;\n self.pools.ledger.reinit();\n'), + # Count the chunks freed although the VMs could not be walked. + 'freed-without-walk': (POOLS, ' );\n 0\n }\n };\n', + ' );\n dropped\n }\n };\n'), # Catch up without the one-VM check. - 'no-activation-guard': (POOLS, '.any(|f| !core::ptr::eq(&*f.job.vm, vm))', '.any(|_| false)'), - # Neither the catch-up's nor the grant's one-VM check (each alone covers a grant). - 'no-same-vm-checks': [(POOLS, '.any(|f| !core::ptr::eq(&*f.job.vm, vm))', '.any(|_| false)'), - (POOLS, 'if vm.slot != ctx || flights.iter().any(|f| !Arc::ptr_eq(&f.job.vm, &vm)) {', - 'if vm.slot != ctx {')], + 'no-activation-guard': (POOLS, '.filter(|f| !core::ptr::eq(&*f.job.vm, vm))', '.filter(|_| false)'), # Back the whole heap when the GPU starts. - 'initial-backs-whole-heap': (POOLS, 'chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?;', - 'chunks.push(PbChunk::new(dev, 0, buffer::MAX_BLOCKS)?, GFP_KERNEL)?;'), - # Growth chunks of 8 MiB. - 'chunks-above-4mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 32;', 'pub(crate) const CHUNK_BLOCKS: u32 = 64;'), + 'initial-backs-whole-heap': (POOLS, 'for _ in 0..Ledger::chunks_for(ledger.initial) {', + 'for _ in 0..growth::MAX_CHUNKS {'), + # Growth chunks of 32 MiB. + 'chunks-above-16mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 128;', 'pub(crate) const CHUNK_BLOCKS: u32 = 256;'), # A firmware-visible cached PB leaf (its drop would ask the firmware to flush). 'pb-leaf-fw-visible': (HW, 'pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c83;', 'pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c03;'), @@ -183,28 +280,43 @@ # The test each mutation must fail, and a piece of its failure message. INTENDED = { - 'drain-until-empty': ('queued_snapshot_keeps_order', 'unbounded answering in one poll pass'), + 'drain-until-empty': ('queued_snapshot_keeps_order', 'replies in the first pass'), 'skip-last-queued': ('queued_snapshot_keeps_order', 'replies in the first pass'), 'return-after-request-error': ('unprocessed_request_does_not_end_the_pass', 'requests taken in the pass'), 'three-refusals': ('default_growth_to_exhaustion', 'want Some(Log("warn"'), 'never-exhausted': ('default_growth_to_exhaustion', 'growth did not end'), 'warn-every-time': ('default_growth_to_exhaustion', 'later request after exhaustion'), - 'no-record-barrier': ('default_growth_to_exhaustion', 'got Some(Producer(1034)), want Some(Sync)'), + 'no-record-barrier': ('default_growth_to_exhaustion', 'got Some(Producer(74)), want Some(Sync)'), 'no-reply-barrier': ('default_growth_to_exhaustion', 'got Some(Reply([7, 1, 0, 5, 0, 256, 3, 1, 0, 0, 0, 0, 0, 0])), want Some(Sync)'), 'soft-boundary-inclusive': ('soft_max_boundaries', 'got Some(Log("info", "")), want Some(Log("dbg", ""))'), 'soft-max-unclamped': ('soft_max_bounds', 'soft_max, param 0'), 'uncapped-ledger': ('constructor_is_the_old_one_capped', 'new(true, '), - 'grant-without-backing': ('growth_backs_before_posting', 'record of block'), - 'back-after-accept': ('allocation_failure_posts_nothing', 'ledger after a backing failure'), - 'catch-up-cursor-before-append': ('partial_catch_up_counts_only_held_aliases', 'count after a failed append'), - 'growth-cursor-before-append': ('append_failure_posts_nothing', 'count differs from held aliases'), - 'keep-unappended-chunk': ('append_failure_posts_nothing', 'chunks after a backing failure'), - 'no-growth-catch-up': ('growth_catches_the_running_vm_up_first', 'running VM not caught up'), + 'grant-without-backing': ('dry_reserve_waits_and_keeps_order', 'record of block'), + 'refill-before-answers': ('allocation_follows_the_replies', 'allocation before a reply'), + 'refill-maps-no-flight': ('growth_backs_before_posting', 'record of block'), + 'no-wait-requeue': ('dry_reserve_waits_and_keeps_order', 'waiting request lost'), + 'refill-failure-keeps-chunk': ('refill_map_failure_drops_the_chunk', 'chunk kept after a failed map'), + 'refill-during-reinit': ('pending_reinit_without_flights_backs_nothing', 'backed during a pending re-initialization'), + 'refill-skip-with-live-render': ('pending_reinit_with_a_live_render_refills', 'growth request stranded'), + 'refill-for-any-flight': ('pending_reinit_with_compute_only_backs_nothing', 'backed for a compute-only flight'), + 'map-without-identity': ('freed_chunks_are_mapped_again', 'alias of another object'), + 'pending-reinit-stops-free': ('client_boundary_reinit_arms_the_long_idle', 'poll woke in a loop'), + 'old-reinit-guards': ('client_boundary_reinit_arms_the_long_idle', 'kept backing not freed'), + 'pending-reinit-arms-nothing': ('client_boundary_reinit_arms_the_long_idle', 'kept backing not freed'), + 'pending-delay-from-drain': ('client_boundary_reinit_arms_the_long_idle', 'poll woke in a loop'), + 'pending-free-at-level-1': ('client_boundary_reinit_at_level_1_keeps_the_backing', 'freed at level 1'), + 'reclaim-pending-not-set': ('client_boundary_reinit_frees_without_a_render', 'kept backing not freed'), + 'free-on-recovery-reinit': ('recovery_reinit_frees_nothing', 'freed for a recovery'), + 'reclaim-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (64 blocks, floor 32)'), + 'drain-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (3000 blocks'), + 'delay-nothing-grown': ('nothing_grown_arms_nothing', 'poll woke in a loop'), + 'reinit-at-idle-reclaim': ('same_client_render_after_1_1s', 'posted count reset before the InitBM (1500)'), + 'reinit-at-any-reclaim': ('same_client_render_after_60ms', 'posted count reset before the InitBM'), + 'freed-without-walk': ('failed_vm_walk_counts_nothing_freed', 'freed counted with the VM walk failed'), 'no-activation-guard': ('catch_up_refuses_flights_of_another_vm', 'catch-up with a flight of another VM'), - 'no-same-vm-checks': ('grant_refuses_flights_of_two_vms', 'grant with flights of two VMs'), 'initial-backs-whole-heap': ('startup_backs_the_initial_blocks', 'startup allocation'), - 'chunks-above-4mib': ('growth_backs_before_posting', 'a growth chunk above 4 MiB'), + 'chunks-above-16mib': ('growth_backs_before_posting', 'a growth chunk above 16 MiB'), 'pb-leaf-fw-visible': ('pb_leaf_is_gpu_only', 'PB leaf not GPU-only'), } assert set(INTENDED) == set(MUTATIONS) @@ -292,6 +404,26 @@ def consts(text, names): pub fn last(&self) -> Option<&T> { self.0.last() } + pub fn truncate(&mut self, n: usize) { + self.0.truncate(n) + } + pub fn reserve(&mut self, n: usize, _: Flags) -> Result { + self.0.reserve(n); + Ok(()) + } + pub fn iter(&self) -> std::slice::Iter<'_, T> { + self.0.iter() + } +} +/// A kernel mutex, single-threaded here. +pub struct Mutex(pub RefCell); +impl Mutex { + pub fn new(value: T) -> Self { + Mutex(RefCell::new(value)) + } + pub fn lock(&self) -> std::cell::RefMut<'_, T> { + self.0.borrow_mut() + } } #[macro_export] macro_rules! static_assert { @@ -311,20 +443,27 @@ def consts(text, names): /// Backing object `id` of this many bytes allocated, or freed (its last reference gone). Alloc(u32, usize), Free(u32), - /// A PB alias of object `id` at GVA in the VM with this context, made, held by the VM - /// (appended to its driver mappings), or unmapped. + /// A PB alias of object `id` at GVA in the VM with this context, made or unmapped. Map(u32, u64, usize, u32), - Append(u32, u64), Unmap(u32, u64), } thread_local! { pub static LOG: RefCell> = RefCell::new(Vec::new()); pub static PARAM: Cell = Cell::new(PARAM_DEFAULT); - /// Failures to inject: the next this many allocations, alias maps, appends fail. + pub static PARAM_BLOCKS: Cell = Cell::new(PB_BLOCKS_DEFAULT); + /// The initial posting and the reclaim values (hw::tuning PB_BLOCKS, PB_RECLAIM, + /// PB_IDLE_MS and PB_RECLAIM_BLOCKS), which a test may vary from the driver's. + pub static PARAM_RECLAIM: Cell = Cell::new(PB_RECLAIM_DEFAULT); + pub static PARAM_IDLE_MS: Cell = Cell::new(PB_IDLE_MS_DEFAULT); + pub static PARAM_RECLAIM_BLOCKS: Cell = Cell::new(PB_RECLAIM_BLOCKS_DEFAULT); + /// The monotonic clock (engine::now_ns()). + pub static NOW: Cell = Cell::new(1_000_000_000); + /// The VM registry walk (vms.open()) fails this many times. + pub static FAIL_OPEN: Cell = Cell::new(0); + /// Failures to inject: the next this many allocations or alias maps fail. pub static FAIL_ALLOC: Cell = Cell::new(0); pub static FAIL_MAP: Cell = Cell::new(0); pub static MAP_OK_BEFORE_FAIL: Cell = Cell::new(0); - pub static FAIL_APPEND: Cell = Cell::new(0); pub static NEXT_ID: Cell = Cell::new(1); } /// Takes one injected failure of `which`. @@ -391,7 +530,7 @@ def consts(text, names): } } -/// The shared layer's VM: kernel aliases at fixed addresses, held by the VM once appended. +/// The shared layer's VM: kernel aliases at fixed addresses. pub mod mmu { use crate::prelude::*; pub struct Prot(pub u64); @@ -414,11 +553,10 @@ def consts(text, names): } pub struct Vm { pub slot: u32, - pub held: std::cell::RefCell>, } impl Vm { pub fn new(slot: u32) -> Self { - Vm { slot, held: Default::default() } + Vm { slot } } pub fn map_at( &self, @@ -439,24 +577,9 @@ def consts(text, names): if !skip && crate::fail(&crate::FAIL_MAP) { return Err(ENOMEM); } - // Leaves already in use: EBUSY (prepare_map). - let end = addr + size as u64; - if self.held.borrow().iter().any(|m| m.gva < end && addr < m.gva + m.size as u64) { - return Err(EBUSY); - } crate::ev(crate::Ev::Map(self.slot, addr, size, gem.id)); Ok(KernelMapping { slot: self.slot, gva: addr, size, prot: prot.0, gem: gem.clone() }) } - pub fn append_driver_mappings(&self, mappings: KVec) -> Result { - if crate::fail(&crate::FAIL_APPEND) { - return Err(ENOMEM); // the mappings drop here - } - for m in mappings.0 { - crate::ev(crate::Ev::Append(self.slot, m.gva)); - self.held.borrow_mut().push(m); - } - Ok(()) - } } } ''' @@ -479,29 +602,74 @@ def consts(text, names): pub struct Init { pub control: Option>, } + /// vm.rs's chunk mapping, its fields as the driver names them. + pub struct HeapMapping { + pub gem: ARef, + pub _mapping: mmu::KernelMapping, + } + pub struct HeapMappings { + pub list: Mutex>, + } pub struct VmState { + pub id: u64, pub slot: u32, pub vm: mmu::Vm, - pub pb_chunks: AtomicUsize, + pub heap: Arc, } impl VmState { pub fn new(slot: u32) -> Arc { - Arc::new(VmState { slot, vm: mmu::Vm::new(slot), pb_chunks: AtomicUsize::new(0) }) + Arc::new(VmState { + // Keep VM identity distinct from the modeled hardware slot. + id: 0x10000 + u64::from(slot), + slot, + vm: mmu::Vm::new(slot), + heap: Arc::new(HeapMappings { list: Mutex::new(KVec(Vec::new())) }), + }) } } pub struct Job { pub vm: Arc, } + impl Job { + pub fn slot(&self) -> u32 { + self.vm.slot + } + } pub struct Flight { pub job: Job, } pub struct Engine { pub flights: Vec, + /// Jobs accepted and not fully published. + pub pending: Vec, + pub frozen: bool, + } + pub struct Recovery { + pub reinit: bool, + pub armed: Option<(usize, u64)>, + pub failed: Option, + } + impl Recovery { + pub fn failed(&self) -> Option { + self.failed + } + } + /// vm.rs's Registry: the VMs whose client handle may still be open. + pub struct Registry { + pub vms: Vec>, + } + impl Registry { + pub fn open(&self) -> Result>> { + if crate::fail(&crate::FAIL_OPEN) { + return Err(ENOMEM); + } + Ok(KVec(self.vms.clone())) + } } pub struct Receipts { pub renders: u32, } - /// A firmware object: the block list, block_ctl or the heap. + /// A firmware object: the block list or block_ctl. pub struct FwObject { pub gva: Option, pub size: usize, @@ -534,6 +702,10 @@ def consts(text, names): pub grown: u64, pub refused: u64, pub bad: u64, + pub waits: u64, + pub refills: u64, + pub reclaims: u64, + pub freed: u64, } pub struct Live { pub block_ctl: FwObject, @@ -546,6 +718,8 @@ def consts(text, names): pub live: Live, pub drm: ARef, pub chunks: KVec, + pub idle: growth::IdleClock, + pub reclaim_pending: bool, } impl Pools { pub fn chunk_count(&self) -> usize { @@ -554,6 +728,10 @@ def consts(text, names): pub fn chunk_capacity(&self) -> usize { self.chunks.capacity() } + /// The backing objects' ids, in heap order. + pub fn chunk_ids(&self) -> Vec { + self.chunks.iter().map(|c| c.backing.gem.id).collect() + } } pub struct Queue(pub std::collections::VecDeque<[u32; 14]>); impl Queue { @@ -567,6 +745,11 @@ def consts(text, names): assert_eq!(i, 0); self.0.pop_front().ok_or(()) } + pub fn insert_within_capacity(&mut self, i: usize, q: [u32; 14]) -> core::result::Result<(), [u32; 14]> { + assert_eq!(i, 0); + self.0.push_front(q); + Ok(()) + } } pub struct Events { pub grow: Queue, @@ -599,6 +782,8 @@ def consts(text, names): pub pools: Pools, pub init: Init, pub engine: Engine, + pub recovery: Recovery, + pub vms: Registry, pub renders: Vec, pub fw: FwModel, } @@ -610,11 +795,12 @@ def consts(text, names): let vm = VmState::new(5); let mut f = Self::without_flights(); f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); + f.vms.vms.push(vm.clone()); f.map_pb(&vm).unwrap(); crate::take(); f } - /// Pools::new's PB backing: the initial chunk, room for every chunk the ledger needs. + /// Pools::new's PB backing: the initial chunks and the reserve, room for the heap. pub fn without_flights() -> Self { let drm = ARef::new(AsahiDevice); let (ledger, chunks) = pools_new_backing(&drm).unwrap(); @@ -623,6 +809,8 @@ def consts(text, names): pools: Pools { ledger, exhausted_logged: false, + idle: growth::IdleClock::default(), + reclaim_pending: false, stats: PbStats::default(), drm, chunks, @@ -636,7 +824,9 @@ def consts(text, names): }, }, init: Init { control: Some(std::sync::Arc::new(Ctl { epoch: 0x1_0000_0003 })) }, - engine: Engine { flights: Vec::new() }, + engine: Engine { flights: Vec::new(), pending: Vec::new(), frozen: false }, + recovery: Recovery { reinit: false, armed: None, failed: None }, + vms: Registry { vms: Vec::new() }, renders: vec![1], fw: FwModel { ctx: 5, @@ -693,10 +883,10 @@ def consts(text, names): use crate::g15::firmware::{Firmware, VmState, Flight, Job, HEAP}; use crate::g15::hw::{client, pte}; use crate::g15::fw::buffer::{BLOCK_BYTES, MAX_BLOCKS}; - use crate::g15::pools::growth::{max_chunks, CHUNK_BLOCKS}; + use crate::g15::pools::growth::{CHUNK_BLOCKS, MAX_CHUNKS}; + use crate::g15::pools::{chunk_gva, CHUNK_BYTES}; use crate::g15::pgtable::Prot; - use crate::{EPROTO, FAIL_ALLOC, FAIL_APPEND, FAIL_MAP}; - use core::sync::atomic::Ordering; + use crate::{EPROTO, FAIL_ALLOC, FAIL_MAP}; use std::sync::Arc; use crate::g15::fw::buffer::{self, grow_reply, BLOCK_LIST_RECORDS, GROW_STEP}; use crate::g15::fw::channels::devctl; @@ -777,7 +967,7 @@ def consts(text, names): fn levels_only(evs: Vec) -> Vec { evs.into_iter() - .filter(|e| !is_backing(e)) + .filter(|e| !is_backing(e) && !matches!(e, Ev::Log(_, m) if m.starts_with("PB heap:"))) .map(|e| match e { Ev::Log(l, _) => Ev::Log(l, String::new()), e => e, @@ -787,6 +977,7 @@ def consts(text, names): /// Growth to the maximum with the firmware asking after every reply, one pass at a time. fn run_to_exhaustion(param: u32) { + defaults(); PARAM.with(|p| p.set(param)); let mut f = Firmware::new(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); @@ -850,9 +1041,11 @@ def consts(text, names): #[test] fn driver_values() { - let l = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + defaults(); + let l = Ledger::new(tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); assert!(tuning::PB_GROW); - assert_eq!((l.initial, l.max, l.posted), (1024, 4095, 1024)); + assert_eq!(crate::PB_BLOCKS_DEFAULT, 64); + assert_eq!((l.initial, l.max, l.posted), (64, 4095, 64)); assert_eq!(PARAM_DEFAULT, 3072); PARAM.with(|p| p.set(PARAM_DEFAULT)); assert_eq!(params::pb_max_blocks(), 3072); @@ -861,12 +1054,21 @@ def consts(text, names): #[test] fn soft_max_bounds() { + defaults(); let f = Firmware::new(); for param in [0, 1, 63, 64, 65, 1023, 1024, 1025, 3071, 3072, 3073, 4094, 4095, 4096, 8192, u32::MAX] { PARAM.with(|p| p.set(param)); assert_eq!(params::pb_max_blocks(), clamp(param, 64, tuning::PB_MAX_BLOCKS), "pb_max_blocks, param {param}"); - // Below the initial 1024 the firmware still gets 1024. - assert_eq!(f.pools.soft_max(), clamp(param, 1024, CAP), "soft_max, param {param}"); + // Below the initial 64 the firmware still gets 64. + assert_eq!(f.pools.soft_max(), clamp(param, 64, CAP), "soft_max, param {param}"); + } + // With a larger initial posting the firmware gets at least that. + crate::PARAM_BLOCKS.with(|b| b.set(1024)); + let f = Firmware::new(); + assert_eq!(f.pools.ledger.initial, 1024); + for param in [0, 64, 1023, 1024, 3072, 4095, u32::MAX] { + PARAM.with(|p| p.set(param)); + assert_eq!(f.pools.soft_max(), clamp(param, 1024, CAP), "soft_max, param {param}, 1024 initial"); } } @@ -877,7 +1079,7 @@ def consts(text, names): #[test] fn soft_max_boundaries() { - // 3074 is reached exactly (1024 + 205 * 10): that answer is not past it. + // 3074 is reached exactly (64 + 301 * 10): that answer is not past it. for param in [0, 1024, 3072, 3074, 3075, 4094, 4095, u32::MAX] { run_to_exhaustion(param); } @@ -890,6 +1092,7 @@ def consts(text, names): f.queue(); f.queue(); take(); + let mapped = f.pools.ledger.backed as u64; f.pass(); let log = take(); let replies: Vec = log @@ -933,7 +1136,7 @@ def consts(text, names): assert_eq!(f.pools.stats.requests, 2, "requests taken in the pass"); assert_eq!(f.pools.stats.bad, 1); assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -5")))); - assert!(log.contains(&Ev::Producer(1024 + GROW_STEP))); + assert!(log.contains(&Ev::Producer(64 + GROW_STEP))); assert!(f.events.grow.is_empty()); } @@ -947,7 +1150,7 @@ def consts(text, names): take(); f.pass(); let log = take(); - assert_eq!(f.pools.ledger.posted, 1024 + 3 * GROW_STEP); + assert_eq!(f.pools.ledger.posted, 64 + 3 * GROW_STEP); assert_eq!((f.pools.stats.grown, f.pools.stats.bad), (3 * GROW_STEP as u64, 0)); // Each failed reply is logged at info level. let info = log.iter().filter(|e| matches!(e, Ev::Log("info", m) if m.contains("DEVCTL 7: -5"))).count(); @@ -980,7 +1183,9 @@ def consts(text, names): assert!(log.iter().all(|e| matches!(e, Ev::Log("warn", _))), "{log:?}"); assert_eq!(log.len(), 4); assert_eq!(f.pools.stats.bad, 4); - assert_eq!(f.pools.ledger, Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS)); + let mut fresh = Ledger::new(tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); + fresh.backed = Ledger::chunks_for(fresh.initial) * CHUNK_BLOCKS; + assert_eq!(f.pools.ledger, fresh); // No render published: not live. f.renders[0] = 0; f.events.grow.0.push_back(good); @@ -998,54 +1203,43 @@ def consts(text, names): } fn is_backing(e: &Ev) -> bool { - matches!(e, Ev::Alloc(..) | Ev::Free(_) | Ev::Map(..) | Ev::Append(..) | Ev::Unmap(..)) - } - - /// A VM's PB aliases: its count, and the aliases it holds, from block 0 in order. - fn assert_cursor(vm: &VmState, chunks: usize) { - let held = vm.vm.held.borrow(); - assert_eq!(vm.pb_chunks.load(Ordering::Relaxed), held.len(), "count differs from held aliases"); - assert!(held.len() <= chunks); - let mut gva = client::PB_HEAP; - for m in held.iter() { - assert_eq!(m.gva, gva, "aliases not contiguous from the heap's start"); - assert_eq!(m.prot, pte::LEAF_CACHED); - gva += m.size as u64; + matches!(e, Ev::Alloc(..) | Ev::Free(_) | Ev::Map(..) | Ev::Unmap(..)) + } + + /// A VM's PB aliases: exactly the first `chunks` chunks, in order, each the whole chunk at + /// its GVA with the PB leaf, each of the chunk's object. + fn assert_cursor(f: &Firmware, vm: &VmState, chunks: usize) { + let heap = vm.heap.list.lock(); + assert_eq!(heap.len(), chunks, "VM aliases differ from the chunks"); + let ids = f.pools.chunk_ids(); + for (i, m) in heap.iter().enumerate() { + assert_eq!(m._mapping.gva, chunk_gva(i), "aliases not contiguous from the heap's start"); + assert_eq!(m._mapping.size, CHUNK_BYTES); + assert_eq!(m._mapping.prot, pte::LEAF_CACHED); + assert_eq!(m.gem.id, ids[i], "alias of another object"); } } /// In the order of `log`: no block record and no producer past the blocks the VM with - /// context `slot` holds aliases for at that point. + /// context `slot` has mapped at that point (`start` blocks before the log). fn assert_posted_backed(log: &[Ev], slot: u32, start: u64) { let mut mapped = start; for e in log { match e { - Ev::Append(s, gva) if *s == slot => { - assert_eq!(*gva, client::PB_HEAP + mapped * BLOCK_BYTES, "append out of order"); + Ev::Map(s, gva, size, _) if *s == slot => { + assert_eq!(*gva, client::PB_HEAP + mapped * BLOCK_BYTES, "map out of order"); + mapped += *size as u64 / BLOCK_BYTES; } - Ev::Map(s, gva, size, _) if *s == slot && *gva == client::PB_HEAP + mapped * BLOCK_BYTES => { - // Counted at its append. - let _ = size; + Ev::Unmap(s, gva) if *s == slot => { + mapped = mapped.min((gva - client::PB_HEAP) / BLOCK_BYTES); } - _ => {} - } - if let Ev::Append(s, gva) = e { - if *s == slot { - let size = log - .iter() - .find_map(|m| match m { - Ev::Map(ms, mg, size, _) if ms == s && mg == gva => Some(*size as u64), - _ => None, - }) - .unwrap(); - mapped += size / BLOCK_BYTES; + Ev::Record(off, _) => { + assert!((*off as u64 / 8) < mapped, "record of block {} with {} mapped", off / 8, mapped); } - } - if let Ev::Record(off, _) = e { - assert!((*off as u64 / 8) < mapped, "record of block {} with {} mapped", off / 8, mapped); - } - if let Ev::Producer(p) = e { - assert!(*p as u64 <= mapped, "producer {} with {} mapped", p, mapped); + Ev::Producer(p) => { + assert!(*p as u64 <= mapped, "producer {} with {} mapped", p, mapped); + } + _ => {} } } } @@ -1071,17 +1265,20 @@ def consts(text, names): let mut f = Firmware::without_flights(); let log = take(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); - assert_eq!(log, vec![Ev::Alloc(1, initial as usize * BLOCK_BYTES as usize)], "startup allocation"); - assert_eq!((f.pools.backed(), f.pools.chunk_count()), (initial, 1)); - assert_eq!(f.pools.chunk_capacity(), max_chunks(initial, max)); - assert_eq!(max_chunks(initial, max), 97); + let chunks = Ledger::chunks_for(initial) as usize; + assert_eq!(chunks, 2); + let want: Vec = (1..=chunks as u32).map(|id| Ev::Alloc(id, CHUNK_BYTES)).collect(); + assert_eq!(log, want, "startup allocation"); + assert_eq!((f.pools.ledger.backed, f.pools.chunk_count()), (chunks as u32 * CHUNK_BLOCKS, chunks)); + assert_eq!(f.pools.chunk_capacity(), MAX_CHUNKS as usize); + assert!(max <= MAX_CHUNKS * CHUNK_BLOCKS); // The first prefix of a VM maps the backing, nothing else. let vm = VmState::new(7); f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); f.map_pb(&vm).unwrap(); - let size = initial as usize * BLOCK_BYTES as usize; - assert_eq!(take(), vec![Ev::Map(7, client::PB_HEAP, size, 1), Ev::Append(7, client::PB_HEAP)]); - assert_cursor(&vm, 1); + let want: Vec = (0..chunks).map(|i| Ev::Map(7, chunk_gva(i), CHUNK_BYTES, i as u32 + 1)).collect(); + assert_eq!(take(), want); + assert_cursor(&f, &vm, chunks); // Mapping again maps nothing. f.map_pb(&vm).unwrap(); assert!(take().is_empty()); @@ -1095,8 +1292,10 @@ def consts(text, names): #[test] fn growth_backs_before_posting() { - PARAM.with(|p| p.set(PARAM_DEFAULT)); + defaults(); let mut f = Firmware::new(); + let start = f.pools.ledger.backed as u64; + let first_chunks = f.pools.chunk_count(); f.fw.ask_after_reply = true; f.queue(); take(); @@ -1107,77 +1306,179 @@ def consts(text, names): assert!(passes <= 1000, "growth did not end"); } let log = take(); - let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); - assert_posted_backed(&log, 5, initial as u64); - // Chunks of at most 32 blocks, allocated only as needed: up to the maximum, no further. + assert_posted_backed(&log, 5, start); + // 16 MiB chunks, backed as the reserve needs them: up to the whole heap, no further. let allocs: Vec = log .iter() .filter_map(|e| if let Ev::Alloc(_, b) = e { Some(*b) } else { None }) .collect(); - assert_eq!(allocs.len() + 1, max_chunks(initial, max)); - assert!(allocs.iter().all(|b| *b <= 4 << 20), "a growth chunk above 4 MiB"); - assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 4 << 20); - assert_eq!(f.pools.backed(), max); + assert!(allocs.iter().all(|b| *b <= 16 << 20), "a growth chunk above 16 MiB"); + assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 16 << 20); + assert_eq!(allocs.len() + first_chunks, MAX_CHUNKS as usize); + assert_eq!(f.pools.ledger.backed, MAX_BLOCKS); + assert_eq!(f.pools.stats.waits, 0); + assert_eq!(f.pools.stats.refills, allocs.len() as u64); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); - assert_cursor(&f.engine.flights[0].job.vm, f.pools.chunk_count()); + assert_cursor(&f, &f.engine.flights[0].job.vm, f.pools.chunk_count()); } - /// A failure while backing posts nothing and answers nothing; the request counter and the - /// VM's count stay; the next request is answered. - fn failure_posts_nothing(which: &'static std::thread::LocalKey>) -> Vec { + #[test] + fn allocation_follows_the_replies() { let mut f = Firmware::new(); - which.with(|c| c.set(1)); + // Below the reserve line: the pass's answers come first, then the chunk. + f.pools.ledger.posted = f.pools.ledger.backed - growth_reserve() + 1; f.queue(); - f.pass(); - let log = take(); - assert_eq!(count(&log, |e| matches!(e, Ev::Record(..) | Ev::Producer(_) | Ev::Reply(_) | Ev::Sync)), 0, - "posted after a backing failure: {log:?}"); - assert_eq!((f.pools.ledger.posted, f.pools.ledger.next_request), (1024, 0), "ledger after a backing failure"); - assert_eq!((f.pools.stats.bad, f.pools.chunk_count()), (1, 1), "chunks after a backing failure"); - assert_cursor(&f.engine.flights[0].job.vm, 1); - assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -12")))); f.queue(); + take(); f.pass(); - let retry = take(); - assert!(retry.contains(&Ev::Producer(1034))); - assert_eq!(f.pools.chunk_count(), 2); - assert_cursor(&f.engine.flights[0].job.vm, 2); - log + let log = take(); + let last_reply = log.iter().rposition(|e| matches!(e, Ev::Reply(_))).expect("replies"); + let first_alloc = log.iter().position(|e| matches!(e, Ev::Alloc(..))).expect("a refill"); + assert!(first_alloc > last_reply, "allocation before a reply: {log:?}"); + assert_eq!(count(&log, |e| matches!(e, Ev::Reply(_))), 2); + assert_eq!(f.pools.stats.refills, 1); + } + + fn growth_reserve() -> u32 { + crate::g15::pools::growth::RESERVE_BLOCKS } #[test] - fn allocation_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_ALLOC); - assert_eq!(count(&log, is_backing), 0); + fn dry_reserve_waits_and_keeps_order() { + let mut f = Firmware::new(); + let start = f.pools.ledger.backed as u64; + FAIL_ALLOC.with(|c| c.set(1000)); + let mut log = Vec::new(); + let mut waited = None; + for _ in 0..40 { + f.queue(); + f.pass(); + let pass = take(); + let replied = pass.iter().any(|e| matches!(e, Ev::Reply(_))); + log.extend(pass); + if !replied { + waited = Some(f.fw.next_counter - 1); + break; + } + } + assert_posted_backed(&log, 5, start); + let waited = waited.expect("no request waited"); + assert_eq!(f.pools.ledger.posted, f.pools.ledger.backed); + let queued: Vec = f.events.grow.0.iter().map(|q| q[buffer::grow_request::COUNTER]).collect(); + assert_eq!(queued, vec![waited], "waiting request lost"); + assert_eq!(f.pools.ledger.next_request, waited); + assert!(f.pools.stats.waits >= 1); + // The allocator recovers: this pass still waits, then backs a chunk; the next answers. + FAIL_ALLOC.with(|c| c.set(0)); + f.pass(); + let log = take(); + assert!(!log.iter().any(|e| matches!(e, Ev::Reply(_)))); + assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 1); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 1); + let mapped = f.pools.ledger.backed as u64; + f.pass(); + let log = take(); + let replies: Vec = log + .iter() + .filter_map(|e| if let Ev::Reply(r) = e { Some(r[grow_reply::COUNTER]) } else { None }) + .collect(); + assert_eq!(replies, vec![waited]); + assert!(f.events.grow.is_empty()); + assert_posted_backed(&log, 5, mapped); } #[test] - fn map_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_MAP); + fn refill_map_failure_drops_the_chunk() { + let mut f = Firmware::new(); + let chunks = f.pools.chunk_count(); + let backed = f.pools.ledger.backed; + f.pools.ledger.posted = backed - growth_reserve() + 1; + FAIL_MAP.with(|c| c.set(1)); + take(); + f.pass(); + let log = take(); + assert_eq!(f.pools.chunk_count(), chunks, "chunk kept after a failed map"); let backing: Vec<&Ev> = log.iter().filter(|e| is_backing(e)).collect(); - assert!(matches!(backing[..], [Ev::Alloc(2, _), Ev::Free(2)]), "{backing:?}"); + assert!(matches!(backing[..], [Ev::Alloc(id, _), Ev::Free(free)] if id == free), "{backing:?}"); + assert_eq!(f.pools.ledger.backed, backed); + assert!(log.iter().any(|e| matches!(e, Ev::Log("warn", m) if m.contains("backing chunk")))); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks); + // The next pass backs it. + f.pass(); + assert_eq!(f.pools.chunk_count(), chunks + 1); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks + 1); } + /// A pending re-initialization with nothing published (a reclaim, or a recovery that + /// failed the flights): the posted count is about to drop, so nothing is backed. #[test] - fn append_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_APPEND); - let backing: Vec<&Ev> = log.iter().filter(|e| is_backing(e)).collect(); + fn pending_reinit_without_flights_backs_nothing() { + let mut f = Firmware::without_flights(); + f.pools.ledger.posted = f.pools.ledger.backed - growth_reserve() + 1; + f.recovery.reinit = true; + take(); + f.pass(); + assert_eq!(count(&take(), is_backing), 0, "backed during a pending re-initialization"); + f.recovery.reinit = false; + f.pass(); + assert_eq!(f.pools.stats.refills, 1); + } + + /// A pending re-initialization with only compute published (no render can ask for + /// growth): nothing is backed. + #[test] + fn pending_reinit_with_compute_only_backs_nothing() { + let mut f = Firmware::new(); + f.renders[0] = 0; + f.pools.ledger.posted = f.pools.ledger.backed; + f.recovery.reinit = true; + take(); + for _ in 0..4 { + f.pass(); + } + assert_eq!(count(&take(), is_backing), 0, "backed for a compute-only flight"); + assert_eq!(f.pools.stats.refills, 0); + } + + /// A render published before the re-initialization was set (the flights behind a fragment + /// kill) still asks for growth: with its reserve used up, the refill must still back + /// chunks, or the request waits until the deadline. + #[test] + fn pending_reinit_with_a_live_render_refills() { + let mut f = Firmware::new(); // one live render + let start = f.pools.ledger.backed as u64; + f.pools.ledger.posted = f.pools.ledger.backed; + f.recovery.reinit = true; + f.queue(); + take(); + for _ in 0..4 { + f.pass(); + } + let s = &f.pools.stats; assert!( - matches!(backing[..], [Ev::Alloc(2, _), Ev::Map(5, _, _, 2), Ev::Unmap(5, _), Ev::Free(2)]), - "{backing:?}" + f.events.grow.is_empty(), + "growth request stranded: posted={} backed={} waits={} refills={}", + f.pools.ledger.posted, + f.pools.ledger.backed, + s.waits, + s.refills ); + // The request waits one pass for the chunk; the grant's pass tops the reserve up. + assert_eq!((s.waits, s.refills, s.requests), (1, 2, 1)); + assert_posted_backed(&take(), 5, start); } #[test] fn transport_failure_keeps_everything() { let mut f = Firmware::new(); + let chunks = f.pools.chunk_count(); f.fw.reply_fails = true; f.queue(); f.pass(); let log = take(); - assert!(log.contains(&Ev::Producer(1034))); - assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (1034, 2)); - assert_cursor(&f.engine.flights[0].job.vm, 2); + assert!(log.contains(&Ev::Producer(74))); + assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (74, chunks)); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); } @@ -1185,9 +1486,10 @@ def consts(text, names): fn other_vms_catch_up_before_they_run() { let mut f = Firmware::new(); let a = f.engine.flights[0].job.vm.clone(); - grow_to(&mut f, 1100); + grow_to(&mut f, 200); take(); - assert_eq!(f.pools.chunk_count(), 4); + let chunks = f.pools.chunk_count(); + assert_eq!(chunks, 3); // A's work retires; a new VM runs next: it maps every chunk first. f.engine.flights.clear(); let b = VmState::new(6); @@ -1195,84 +1497,53 @@ def consts(text, names): f.fw.ctx = 6; f.map_pb(&b).unwrap(); let log = take(); - assert_eq!(count(&log, |e| matches!(e, Ev::Append(6, _))), 4); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(6, ..))), chunks); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 0); - assert_cursor(&b, 4); + assert_cursor(&f, &b, chunks); // B grows the backing; A, behind, catches up when it runs again. - grow_to(&mut f, 1200); + grow_to(&mut f, 300); let log = take(); - assert_posted_backed(&log, 6, 1120); + assert_posted_backed(&log, 6, chunks as u64 * CHUNK_BLOCKS as u64); assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 0); + assert_eq!(f.pools.chunk_count(), chunks + 1); f.engine.flights.clear(); f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); f.map_pb(&a).unwrap(); - assert_cursor(&a, f.pools.chunk_count()); - assert_eq!(a.pb_chunks.load(Ordering::Relaxed), f.pools.chunk_count()); + assert_cursor(&f, &a, f.pools.chunk_count()); } #[test] - fn growth_catches_the_running_vm_up_first() { + fn partial_catch_up_keeps_the_aliases_made() { let mut f = Firmware::new(); - let a = f.engine.flights[0].job.vm.clone(); - // B grows the backing while A is idle. - f.engine.flights.clear(); - let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - f.map_pb(&b).unwrap(); - f.fw.ctx = 6; - grow_to(&mut f, 1100); - // A runs again; its growth request maps what it lacks before backing more. - f.engine.flights.clear(); - f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); - f.fw.ctx = 5; - take(); - grow_to(&mut f, 1130); - let log = take(); - assert_eq!(a.vm.held.borrow().len(), f.pools.chunk_count(), "running VM not caught up"); - assert_cursor(&a, f.pools.chunk_count()); - assert_posted_backed(&log, 5, 1024); - } - - #[test] - fn partial_catch_up_counts_only_held_aliases() { - let mut f = Firmware::new(); - grow_to(&mut f, 1100); + grow_to(&mut f, 200); + let chunks = f.pools.chunk_count(); f.engine.flights.clear(); let b = VmState::new(6); f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); take(); // The third alias cannot be made: two held. - FAIL_MAP.with(|c| c.set(0)); crate::MAP_OK_BEFORE_FAIL.with(|c| c.set(2)); FAIL_MAP.with(|c| c.set(1)); assert_eq!(f.map_pb(&b), Err(ENOMEM)); - assert_cursor(&b, 4); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2); - // The third alias is made but cannot be held: dropped, still two. - FAIL_APPEND.with(|c| c.set(1)); - assert_eq!(f.map_pb(&b), Err(ENOMEM)); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2, "count after a failed append"); - assert!(take().iter().any(|e| matches!(e, Ev::Unmap(6, _)))); - assert_cursor(&b, 4); + assert_cursor(&f, &b, 2); f.map_pb(&b).unwrap(); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 4); - assert_cursor(&b, 4); + assert_cursor(&f, &b, chunks); } /// The ledger reset of Pools::init(), not a full recovery. #[test] fn ledger_reset_reuses_the_backing() { let mut f = Firmware::new(); - grow_to(&mut f, 1300); + grow_to(&mut f, 300); let chunks = f.pools.chunk_count(); take(); f.reset_ledger(); - assert_eq!(f.pools.ledger.posted, 1024); - grow_to(&mut f, 1300); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 300); let log = take(); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..) | Ev::Map(..))), 0); assert_eq!(f.pools.chunk_count(), chunks); - assert_posted_backed(&log, 5, f.pools.backed() as u64); + assert_posted_backed(&log, 5, f.pools.ledger.backed as u64); } #[test] @@ -1287,36 +1558,421 @@ def consts(text, names): assert_eq!(f.pools.stats.bad, 1); } + /// Every PB parameter at its default. libtest gives each test its own thread, but a test + /// that sets one starts from the defaults all the same. + fn defaults() { + PARAM.with(|p| p.set(PARAM_DEFAULT)); + crate::PARAM_BLOCKS.with(|b| b.set(crate::PB_BLOCKS_DEFAULT)); + crate::PARAM_RECLAIM.with(|r| r.set(crate::PB_RECLAIM_DEFAULT)); + crate::PARAM_IDLE_MS.with(|r| r.set(crate::PB_IDLE_MS_DEFAULT)); + crate::PARAM_RECLAIM_BLOCKS.with(|r| r.set(crate::PB_RECLAIM_BLOCKS_DEFAULT)); + } + + /// A prefix of `vm` published as engine.rs does it: publish_prefix_into() first catches the + /// VM up (map_pb(), before the flight is added), and a render prefix with a + /// re-initialization pending does the InitBM (Pools::init()'s ledger reset) and clears + /// `reinit` at its publication. `render`: the prefix has renders. + fn publish(f: &mut Firmware, vm: &Arc, render: bool) { + f.map_pb(vm).unwrap(); + if render && f.recovery.reinit { + f.reset_ledger(); + f.recovery.reinit = false; + } + f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); + let i = f.engine.flights.len() - 1; + if f.renders.len() <= i { + f.renders.resize(i + 1, 0); + } + f.renders[i] = render as u32; + f.fw.ctx = vm.slot; + f.note_idle(); + } + + /// What a reclaim leaves behind in a VM that was not walked (a closed VM no longer in the + /// registry, whose accepted jobs may still run): its aliases of freed chunks go, and chunks + /// backed again at the same GVA are mapped from their new objects at its next publication. + /// The reclaims are the actual idle reclaim; the posted count stays until the InitBM. #[test] - fn catch_up_refuses_flights_of_another_vm() { + fn freed_chunks_are_mapped_again() { + defaults(); let mut f = Firmware::new(); let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(chunks > 2); + // A's work retires and A leaves the registry; the idle reclaim frees the chunks above + // the kept two without walking A. + f.engine.flights.clear(); + f.vms.vms.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + take(); + // B runs: its first render re-initializes; it grows the backing again, new objects at + // the old GVAs. let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - assert_eq!(f.map_pb(&b), Err(EPROTO), "catch-up with a flight of another VM"); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 0); - assert_eq!(count(&take(), is_backing), 0); - // A count past the chunks. - f.engine.flights.pop(); - a.pb_chunks.store(f.pools.chunk_count() + 1, Ordering::Relaxed); - assert_eq!(f.map_pb(&a), Err(EPROTO), "catch-up with a count past the chunks"); + f.vms.vms.push(b.clone()); + publish(&mut f, &b, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 1100); + assert_eq!(f.pools.chunk_count(), chunks); + // A runs again: its stale aliases are replaced by the new objects'. + f.engine.flights.clear(); + take(); + publish(&mut f, &a, true); + let log = take(); + assert_cursor(&f, &a, chunks); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), chunks - 2); + // Another idle reclaim walks B, not A: with fewer chunks than A maps, A's aliases past + // the backing go at its next catch-up. + f.engine.flights.clear(); + let t1 = now(); + check_at(&mut f, t1); + check_at(&mut f, t1 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + assert_cursor(&f, &b, 2); + f.map_pb(&a).unwrap(); + assert_cursor(&f, &a, 2); } + /// A reclaim whose walk of the VM registry fails: the chunks leave the backing, but a VM's + /// aliases still hold their pages, so nothing is counted freed; the VM drops its stale + /// aliases at its next publication, and only then are the pages released. #[test] - fn grant_refuses_flights_of_two_vms() { + fn failed_vm_walk_counts_nothing_freed() { + defaults(); let mut f = Firmware::new(); - let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - f.renders.push(1); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1500); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + take(); + crate::FAIL_OPEN.with(|c| c.set(1)); + check_at(&mut f, t0 + 1_000_000_000); + let log = take(); + assert_eq!(f.pools.chunk_count(), 2); + assert_eq!(f.pools.stats.freed, 0, "freed counted with the VM walk failed"); + assert!(log.iter().any(|e| matches!(e, Ev::Log("warn", m) if m.contains("could not walk the VMs")))); + assert!(log.iter().any(|e| matches!(e, Ev::Log("info", m) if m.contains(" 0 chunks freed")))); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), 0, "pages released while aliases hold them"); + assert_eq!(a.heap.list.lock().len(), chunks); + // A's next publication drops the stale aliases: the pages go. + publish(&mut f, &a, true); + let log = take(); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + assert_cursor(&f, &a, 2); + } + + /// One client idles 60 ms after a heavy scene (more than half the maximum posted) and + /// renders again: the client-boundary re-initialization was set at 50 ms with the backing + /// kept, the posted count stays until the render's InitBM, which posts the initial blocks + /// again, and the scene grows again from the kept backing with nothing allocated or mapped. + #[test] + fn same_client_render_after_60ms() { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 60_000_000); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!( + (f.pools.ledger.posted, f.pools.chunk_count()), + (posted, chunks), + "posted count reset before the InitBM" + ); + take(); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + assert!(!f.recovery.reinit && !f.pools.reclaim_pending); + grow_to(&mut f, 2100); + let log = take(); + assert_eq!(count(&log, is_backing), 0, "backing changed: {:?}", log.iter().filter(|e| is_backing(e)).collect::>()); + assert_posted_backed(&log, 5, f.pools.ledger.backed as u64); + } + + /// One client idles 1.1 s and renders again, after a scene above the reclaim floor (the + /// idle reclaim at 1 s) or above half the maximum (the client boundary at 50 ms, then the + /// free at 1 s): the backing above the kept chunks was freed and unmapped from the client's + /// VM, the posted count stays until the render's InitBM, and the scene grows again with + /// every chunk backed and mapped before a record or producer names it. + #[test] + fn same_client_render_after_1_1s() { + for target in [1500, 2100] { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, target); + let posted = f.pools.ledger.posted; + f.engine.flights.clear(); + let t0 = now(); + for dt in [0, 50_000_000, 1_000_000_000, 1_100_000_000] { + check_at(&mut f, t0 + dt); + } + assert_eq!(f.pools.chunk_count(), 2, "not freed ({target})"); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM ({target})"); + assert_cursor(&f, &a, 2); + take(); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, target); + let log = take(); + assert_posted_backed(&log, 5, 2 * CHUNK_BLOCKS as u64); + assert_cursor(&f, &a, f.pools.chunk_count()); + } + } + + /// A compute-only submission between a level-2 reclaim and the next render's InitBM: its + /// publication maps nothing, its passes back and post nothing (the posted count is still + /// the old one, above the kept backing), and a growth request naming its context is left + /// unanswered; once it retires, the render after it re-initializes as usual. + #[test] + fn compute_between_reclaim_and_initbm() { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1500); + let posted = f.pools.ledger.posted; + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + let bad = f.pools.stats.bad; + take(); + publish(&mut f, &a, false); f.queue(); - f.pass(); + for _ in 0..4 { + f.pass(); + } + let log = take(); + let touched: Vec<&Ev> = log + .iter() + .filter(|e| is_backing(e) || matches!(e, Ev::Record(..) | Ev::Producer(_) | Ev::Reply(_))) + .collect(); + assert!(touched.is_empty(), "compute-only flight backed or posted: {touched:?}"); + assert!(f.recovery.reinit && f.pools.ledger.posted == posted); + assert_eq!(f.pools.stats.bad, bad + 1); + // The compute retires; the check finds nothing more to do; the next render re-inits. + f.engine.flights.clear(); + check_at(&mut f, t0 + 2_000_000_000); + assert_eq!(count(&take(), is_backing), 0); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 300); + assert_posted_backed(&take(), 5, 2 * CHUNK_BLOCKS as u64); + } + + fn now() -> u64 { + crate::NOW.with(|n| n.get()) + } + + /// The idle clock at `t` and the reclaim check at that time, as a job arrival or an idle + /// poll pass runs it. + fn check_at(f: &mut Firmware, t: u64) { + crate::NOW.with(|n| n.set(t)); + f.note_idle(); + f.reclaim_check(); + } + + /// A heavy client (more than half the maximum posted) exits and nothing renders after it: + /// the 50 ms client-boundary re-initialization keeps the backing, and a check after the + /// long idle frees the backing above the kept chunks. The posted count stays until the next + /// render's InitBM. + #[test] + fn client_boundary_reinit_frees_without_a_render() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(posted > 2047 && chunks > 2); + // The pass that retires the last flight notes the idle. + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + take(); + check_at(&mut f, t0 + 50_000_000); + assert!(f.recovery.reinit && f.pools.chunk_count() == chunks, "client-boundary re-initialization"); + check_at(&mut f, t0 + 999_999_999); + assert_eq!(f.pools.chunk_count(), chunks, "freed before the long idle"); + check_at(&mut f, t0 + 1_000_000_000); + let log = take(); + assert_eq!(f.pools.chunk_count(), 2, "kept backing not freed: {} chunks", f.pools.chunk_count()); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!(f.pools.stats.reclaims, 1); + assert_eq!(f.pools.ledger.backed, 2 * CHUNK_BLOCKS); + assert_eq!(f.pools.stats.freed, (chunks - 2) as u64); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + // Later checks find nothing more to do. + check_at(&mut f, t0 + 100_000_000_000); + assert!(take().is_empty()); + } + + /// With a reclaim level of 1 (tuning::PB_RECLAIM is 2) the client-boundary re-initialization keeps the backing for good. + #[test] + fn client_boundary_reinit_at_level_1_keeps_the_backing() { + defaults(); + crate::PARAM_RECLAIM.with(|r| r.set(1)); + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 50_000_000); + assert!(f.recovery.reinit); + assert_eq!(f.reclaim_delay_ns(), None, "armed at level 1"); + check_at(&mut f, t0 + 5_000_000_000); + assert_eq!(f.pools.chunk_count(), chunks, "freed at level 1"); + assert_eq!(f.pools.stats.freed, 0); + } + + /// Nothing grown since the initialization: an initial posting above half the maximum + /// (an initial posting of 3000), or a floor below it (a reclaim floor of 32): no + /// re-initialization at any idle. + #[test] + fn nothing_grown_reinitializes_nothing() { + for (blocks, floor) in [(64, 32), (3000, 1024)] { + defaults(); + crate::PARAM_BLOCKS.with(|b| b.set(blocks)); + crate::PARAM_RECLAIM_BLOCKS.with(|b| b.set(floor)); + let mut f = Firmware::new(); + assert_eq!(f.pools.ledger.posted, blocks); + f.engine.flights.clear(); + let t0 = now(); + for dt in [0, 50_000_000, 1_000_000_000, 5_000_000_000] { + check_at(&mut f, t0 + dt); + } + assert!( + !f.recovery.reinit && f.pools.stats.reclaims == 0, + "re-initialization with nothing grown ({blocks} blocks, floor {floor})" + ); + } + } + + /// A recovery's re-initialization (not a reclaim's) is left alone: nothing freed. + #[test] + fn recovery_reinit_frees_nothing() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + f.recovery.reinit = true; + let t0 = now(); + check_at(&mut f, t0); + assert_eq!(f.reclaim_delay_ns(), None, "armed for a recovery's re-initialization"); + check_at(&mut f, t0 + 5_000_000_000); + assert_eq!(f.pools.chunk_count(), chunks, "freed for a recovery's re-initialization"); + assert_eq!(f.pools.stats.reclaims, 0); + } + + /// The idle poll work as poll.rs drives it: a pass with nothing published notes the idle + /// and runs the reclaim check; an idle pass is re-armed only for reclaim_delay_ns(), + /// rounded up to whole ms, plus one 4 ms jiffy. Returns the wake times. + fn idle_polls(f: &mut Firmware, limit: usize) -> Vec { + let mut wakes = Vec::new(); + loop { + f.note_idle(); + f.reclaim_check(); + let Some(ns) = f.reclaim_delay_ns() else { + return wakes; + }; + assert!(wakes.len() < limit, "poll woke in a loop: {} wakes, next in {ns} ns", wakes.len()); + let t = now() + ns.div_ceil(1_000_000) * 1_000_000 + 4_000_000; + crate::NOW.with(|n| n.set(t)); + wakes.push(t); + } + } + + /// The same exit with the idle poll work: the poll is armed for the 50 ms + /// re-initialization, then for the rest of the long idle (not the expired 50 ms), where the + /// kept backing is freed; then nothing is armed. + #[test] + fn client_boundary_reinit_arms_the_long_idle() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(posted > 2047 && chunks > 2); + // The pass that retires the last flight notes the idle. + f.engine.flights.clear(); + f.note_idle(); + let t0 = now(); + take(); + let wakes = idle_polls(&mut f, 8); let log = take(); assert_eq!( - count(&log, |e| is_backing(e) || matches!(e, Ev::Reply(_) | Ev::Producer(_) | Ev::Record(..))), - 0, - "grant with flights of two VMs" + f.pools.chunk_count(), + 2, + "kept backing not freed: {} chunks, wakes {:?} ms", + f.pools.chunk_count(), + wakes.iter().map(|t| (t - t0) / 1_000_000).collect::>() ); - assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -71")))); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!(f.pools.stats.reclaims, 1); + assert_eq!(f.pools.ledger.backed, 2 * CHUNK_BLOCKS); + assert_eq!(f.pools.stats.freed, (chunks - 2) as u64); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + // Two wakes: the re-initialization after 50 ms, the free after the long idle. + let ms: Vec = wakes.iter().map(|t| (t - t0) / 1_000_000).collect(); + assert_eq!(ms, vec![54, 1004], "wakes (ms after the idle began)"); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + // Idle much longer, or a job arriving: nothing more is due. + crate::NOW.with(|n| n.set(t0 + 100_000_000_000)); + f.reclaim_check(); + assert_eq!(f.reclaim_delay_ns(), None); + assert!(take().is_empty()); + } + + /// Nothing grown: the idle poll work arms nothing. + #[test] + fn nothing_grown_arms_nothing() { + for (blocks, floor) in [(64, 32), (3000, 1024)] { + defaults(); + crate::PARAM_BLOCKS.with(|b| b.set(blocks)); + crate::PARAM_RECLAIM_BLOCKS.with(|b| b.set(floor)); + let mut f = Firmware::new(); + f.engine.flights.clear(); + let wakes = idle_polls(&mut f, 8); + assert!(wakes.is_empty(), "armed with nothing grown ({blocks} blocks, floor {floor}): {} wakes", wakes.len()); + assert!(!f.recovery.reinit); + } + } + + /// An idle reclaim below half the maximum frees at the long idle; with its re-initialization + /// pending nothing more is armed. + #[test] + fn idle_reclaim_arms_nothing_after() { + let mut f = Firmware::new(); + grow_to(&mut f, 1500); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + f.note_idle(); + let t0 = now(); + let wakes = idle_polls(&mut f, 8); + let ms: Vec = wakes.iter().map(|t| (t - t0) / 1_000_000).collect(); + assert_eq!(ms, vec![1004]); + assert_eq!((f.pools.chunk_count(), f.pools.stats.freed), (2, (chunks - 2) as u64)); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + } + + #[test] + fn catch_up_refuses_flights_of_another_vm() { + let mut f = Firmware::new(); + let b = VmState::new(6); + f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); + assert_eq!(f.map_pb(&b), Err(EPROTO), "catch-up with a flight of another VM"); + assert_eq!(b.heap.list.lock().len(), 0); + assert_eq!(count(&take(), is_backing), 0); } #[test] @@ -1348,7 +2004,7 @@ def consts(text, names): // Only inputs the old constructor took to the whole 4096-block heap changed. assert!(changed > 0 && changed < cases); // The driver's values are below the cap, so its ledger is the old one. - let d = (tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let d = (tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); let (old, new) = (OldLedger::new(d.0, d.1, d.2), Ledger::new(d.0, d.1, d.2)); if d.2 <= CAP { assert_eq!((old.initial, old.max, old.posted), (new.initial, new.max, new.posted)); @@ -1397,7 +2053,7 @@ def read(path, ref=None): raise SystemExit('embedded Ledger::new() differs from ' + OLD_REF) def run(mutation): - paths = (POOLS, GROWTH, BUFFER, CHANNELS, INITDATA, PARAMS, HW, CORE, ENGINE, PGTABLE) + paths = (POOLS, GROWTH, BUFFER, CHANNELS, INITDATA, PARAMS, HW, CORE, ENGINE, PGTABLE, VM) src = {path: read(path) for path in paths} if mutation: spec = {**MUTATIONS, **CONTROLS}[mutation] @@ -1407,15 +2063,28 @@ def run(mutation): src[path] = src[path].replace(old, new) pools, hw = src[POOLS], src[HW] bodies = [item(pools, s) for s in (' pub(crate) fn answer_growth(&mut self) -> Result {', - ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', + ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', ' pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result {', - ' fn running_vm(&self, ctx: u32) -> Result> {', - ' fn back_pb(&mut self, end: u32, ctx: u32) -> Result {')] + ' pub(crate) fn refill_backing(&mut self) {', + ' pub(crate) fn note_idle(&mut self) {', + ' pub(crate) fn reclaim_check(&mut self) {', + ' fn free_backing(&mut self) -> usize {', + ' pub(crate) fn reclaim_delay_ns(&self) -> Option {')] soft_max = item(pools, ' pub(crate) fn soft_max(&self) -> u32 {') - backed = item(pools, ' pub(crate) fn backed(&self) -> u32 {') + pools_methods = (item(pools, ' fn map_heap(&self, vm: &VmState) -> Result {') + '\n' + + item(pools, ' fn free_chunks(&mut self, keep: usize) -> usize {') + '\n' + + item(pools, ' fn back_chunk(&mut self) -> Result> {')) chunk = item(pools, 'struct PbChunk {') + '\n' + item(pools, 'impl PbChunk {') + chunk_geometry = (re.search(r'^pub\(crate\) const CHUNK_BYTES: usize = [^;]+;', pools, re.M).group(0) + '\n' + + item(pools, 'pub(crate) const fn chunk_gva(i: usize) -> u64 {')) + vm_src = src[VM] + vm_methods = '\n'.join(item(vm_src, s) for s in ( + ' fn map_one(&self, heap: &mut KVec, i: usize, gem: ARef) -> Result {', + ' pub(crate) fn map_heap(&self, gems: impl Iterator>) -> Result {', + ' pub(crate) fn map_chunk(&self, i: usize, gem: ARef) -> Result {', + ' pub(crate) fn unmap_heap_above(&self, keep: usize) {')) start = pools.index(POOLS_NEW_BACKING[0]) - end = pools.index('\n', pools.index(POOLS_NEW_BACKING[1], start)) + 1 + end = pools.index(POOLS_NEW_BACKING[1], start) pools_new_backing = pools[start:end] if pools.count(PB_LEAF_ASSERT) != 1: raise SystemExit('pools.rs does not assert the PB leaf GPU-only') @@ -1437,32 +2106,64 @@ def run(mutation): if init.count(line) != 1: raise SystemExit('Pools::init() does not hold: ' + line) reset = '\n'.join('p.' + line[len('self.'):] for line in LEDGER_RESET) - pb_max_blocks = item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> u32 {') + pb_max_blocks = item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> ') + # The initial posting and the reclaim values are fixed driver values (hw::tuning), not + # module parameters: production's accessors must return exactly those constants. The + # harness's accessors read cells that start at them, so a test can vary them. + fixed = {'pb_blocks': ('u32', 'super::hw::tuning::PB_BLOCKS', 'PARAM_BLOCKS'), + 'pb_reclaim': ('u32', 'super::hw::tuning::PB_RECLAIM', 'PARAM_RECLAIM'), + 'pb_idle_ns': ('u64', 'super::hw::tuning::PB_IDLE_MS as u64 * 1_000_000', 'PARAM_IDLE_MS'), + 'pb_reclaim_blocks': ('u32', 'super::hw::tuning::PB_RECLAIM_BLOCKS', 'PARAM_RECLAIM_BLOCKS')} + for name, (ty, body, cell) in fixed.items(): + got = item(src[PARAMS], f'pub(crate) fn {name}() -> ') + want = f'pub(crate) fn {name}() -> {ty} {{\n {body}\n}}' + if got.strip() != want: + raise SystemExit(f'params.rs {name}() is not the fixed tuning value: {got!r}') + value = f'crate::{cell}.with(|p| p.get())' + if name == 'pb_idle_ns': + value = f'{value} as u64 * 1_000_000' + pb_max_blocks += f'\npub(crate) fn {name}() -> {ty} {{\n {value}\n}}\n' + tuning_hw = item(hw, 'pub(crate) mod tuning {') + reclaim_defaults = {} + for name, const in (('PB_RECLAIM', 'PB_RECLAIM_DEFAULT'), ('PB_IDLE_MS', 'PB_IDLE_MS_DEFAULT'), + ('PB_RECLAIM_BLOCKS', 'PB_RECLAIM_BLOCKS_DEFAULT')): + m = re.search(rf'pub\(crate\) const {name}: u32 = (\d+);', tuning_hw) + if not m: + raise SystemExit(f'tuning::{name} not found') + reclaim_defaults[const] = int(m.group(1)) m = re.search(r'g15_pb_max_blocks: u32 \{\s*default: (\d+),', src[CORE]) if not m: raise SystemExit('g15_pb_max_blocks default not found') param_default = int(m.group(1)) + m = re.search(r'pub\(crate\) const PB_BLOCKS: u32 = (\d+);', tuning_hw) + if not m: + raise SystemExit('tuning::PB_BLOCKS not found') + blocks_default = int(m.group(1)) sizes = consts(item(hw, 'pub(crate) mod sizes {'), ['PB_BLOCKS']) - tuning = consts(item(hw, 'pub(crate) mod tuning {'), ['PB_GROW', 'PB_BLOCKS', 'PB_MAX_BLOCKS']) + tuning = consts(item(hw, 'pub(crate) mod tuning {'), ['PB_GROW', 'PB_MAX_BLOCKS']) client = consts(item(hw, 'pub(crate) mod client {'), ['PB_HEAP', 'PB_BLOCK_LIST']) pte = consts(item(hw, 'pub(crate) mod pte {'), ['LEAF_CACHED']) control = consts(item(src[INITDATA], 'pub(crate) mod control {'), ['HALT_COUNT']) growth = src[GROWTH] clamp_fn = item(growth, 'const fn clamp(v: u32, lo: u32, hi: u32) -> u32 {') - tested = (''.join(bodies) + soft_max + backed + chunk + pools_new_backing + pb_max_blocks + growth + + tested = (''.join(bodies) + soft_max + pools_methods + chunk + chunk_geometry + vm_methods + + pools_new_backing + pb_max_blocks + growth + src[BUFFER] + sizes + tuning + pte + prot) dir = out / (mutation or 'actual') dir.mkdir(exist_ok=True) for path in (GROWTH, BUFFER, CHANNELS): (dir / Path(path).name).write_text(src[path]) code = (PREFIX + f'pub const PARAM_DEFAULT: u32 = {param_default};\n' + + f'pub const PB_BLOCKS_DEFAULT: u32 = {blocks_default};\n' + + ''.join(f'pub const {k}: u32 = {v};\n' for k, v in reclaim_defaults.items()) + 'pub mod g15 {\n' ' pub mod fw {\n' f' #[path = "{dir / "channels.rs"}"] pub mod channels;\n' f' #[path = "{dir / "buffer.rs"}"] pub mod buffer;\n' ' pub mod initdata { pub mod control {\n' + control + '\n} }\n' ' }\n' - f' pub mod pools {{ #[path = "{dir / "growth.rs"}"] pub mod growth; }}\n' + f' pub mod pools {{ #[path = "{dir / "growth.rs"}"] pub mod growth;\n' + ' use super::fw::buffer;\n use super::hw::client;\n' + chunk_geometry + '\n }\n' ' pub mod hw {\n' ' pub mod sizes {\n' + sizes + '\n}\n' ' pub mod tuning {\n' + tuning + '\n}\n' @@ -1471,6 +2172,7 @@ def run(mutation): ' }\n' ' pub mod pgtable {\n' + pg_consts + '\n' + prot + '\n }\n' ' pub mod boot {\n pub fn eproto() -> crate::Error { crate::EPROTO }\n }\n' + ' pub mod engine {\n pub fn now_ns() -> u64 { crate::NOW.with(|n| n.get()) }\n }\n' ' /// The checked backing allocation, failing where a test injects it.\n' ' pub mod object {\n' ' use crate::prelude::*;\n' @@ -1495,19 +2197,20 @@ def run(mutation): ' use super::fw::buffer::{self, block_ctl};\n' ' use super::fw::channels::DevCtl;\n' ' use super::fw::initdata::control;\n' - ' use super::hw::{client, pte, tuning};\n' - ' use super::pools::growth::{self, Ledger};\n' + ' use super::hw::{self, client, pte, tuning};\n' + ' use super::pools::growth::{self, Answer, Ledger};\n' + ' use super::pools::{chunk_gva, CHUNK_BYTES};\n' ' use super::object::{self, HeldPages};\n' ' use crate::driver::AsahiDevice;\n' - ' use crate::{gem, mmu, ARef};\n' - ' use core::sync::atomic::{AtomicUsize, Ordering};\n' + ' use crate::{gem, mmu, ARef, Mutex};\n' ' use std::sync::Arc;\n' + MODEL.replace('p.RESET', reset).replace('POOLS_NEW_BACKING', '') + chunk + '\n' ' /// Pools::new\'s PB backing, verbatim.\n' ' pub fn pools_new_backing(dev: &AsahiDevice) -> Result<(Ledger, KVec)> {\n' + pools_new_backing + ' Ok((ledger, chunks))\n }\n' - ' impl Pools {\n' + soft_max + '\n' + backed + '\n }\n' + ' impl Pools {\n' + soft_max + '\n' + pools_methods + '\n }\n' + ' impl VmState {\n' + vm_methods + '\n }\n' ' impl Firmware {\n' + '\n'.join(bodies) + '\n }\n' ' }\n' '}\n' + TESTS) @@ -1547,7 +2250,8 @@ def no_core(): result = run(a.mutation) receipt = {'compiler': compiler, 'rustc_version': rustc_version, 'ref': a.ref, 'old_ref': OLD_REF, 'old_reference_checked_against_git': old_git is not None, **result, - 'scope': 'Actual answer_growth/answer_request/soft_max/pb_max_blocks, pools/growth.rs, ' + 'scope': 'Actual answer_growth/answer_request/map_pb/refill_backing/soft_max/pb_max_blocks, ' + 'the PB chunk and VM heap-mapping bodies, pools/growth.rs, ' 'fw/buffer.rs and fw/channels.rs over a model of the firmware objects with an ' 'ordered access log, and Ledger::new against its body at ' f'{OLD_REF[:12]}. Not kernel, firmware or GPU execution, timing, or recovery.'} diff --git a/tools/asahi/g15-probe-init-tests.py b/tools/asahi/g15-probe-init-tests.py index 65cc9c0b69c9f..e7617fba021c2 100644 --- a/tools/asahi/g15-probe-init-tests.py +++ b/tools/asahi/g15-probe-init-tests.py @@ -193,6 +193,8 @@ def item(text, marker): mod boot { pub mod glue { pub fn retained() -> bool { crate::RETAINED.with(core::cell::Cell::get) } pub unsafe fn asahi_g15_reprobe() { crate::activation_reprobe(); } + // Reads the device tree and logs; no driver state. + pub unsafe fn asahi_g15_report_loader_gpu() {} }} ''' diff --git a/tools/asahi/g15-retention-tests.py b/tools/asahi/g15-retention-tests.py index 129567ab373e0..f5c8c8dce3294 100644 --- a/tools/asahi/g15-retention-tests.py +++ b/tools/asahi/g15-retention-tests.py @@ -34,6 +34,8 @@ def item(text, marker): use std::{cell::{Cell, RefCell, RefMut}, mem::MaybeUninit, ops::{Deref, DerefMut}, sync::{Arc, atomic::{AtomicBool, Ordering}}}; type Error = i32; +trait FromErrno { fn from_errno(value: i32) -> Self; } +impl FromErrno for Error { fn from_errno(value: i32) -> Self { value } } type Result = std::result::Result; const GFP_KERNEL: u32 = 0; const EINVAL: Error = -22; @@ -43,6 +45,7 @@ def item(text, marker): static FAIL: Cell = const { Cell::new(0) }; static OWNER_DROPS: Cell = const { Cell::new(0) }; static BUFFER_DROPS: Cell = const { Cell::new(0) }; + static PMP_CALLS: Cell = const { Cell::new(0) }; } fn step(site: u32) -> Result { if FAIL.get() & (1 << site) != 0 { Err(-100 - site as i32) } else { Ok(()) } @@ -72,8 +75,13 @@ def item(text, marker): impl DerefMut for KBox { fn deref_mut(&mut self) -> &mut T { &mut self.0 } } struct AsahiDevice; impl AsahiDevice { fn as_ref(&self) -> &Self { self } } -struct Config; -static CONFIG: Config = Config; +mod hw { + #[derive(PartialEq)] + pub enum Platform { Adt, AdtWithPmp } +} +struct Config { platform: hw::Platform } +static CONFIG: Config = Config { platform: hw::Platform::Adt }; +static PMP_CONFIG: Config = Config { platform: hw::Platform::AdtWithPmp }; mod mmu { pub struct Uat; } mod regs { pub struct Identity; @@ -152,6 +160,12 @@ def item(text, marker): } mod boot { use super::*; + pub mod glue { + pub unsafe fn asahi_g15_pmp_ready() -> i32 { + super::PMP_CALLS.set(super::PMP_CALLS.get() + 1); + match super::step(13) { Ok(()) => 0, Err(e) => e } + } + } use super::{object::Objects, initdata::InitData, channel::Channels, event::Events}; pub struct Session { _owner: Owner, uat: Arc, regs: regs::Regs } impl Session { @@ -185,7 +199,9 @@ def item(text, marker): let boot::Booted { session, init, objects, channels, events, .. } = booted(); boot::Orphan { session, _objects: objects, channels: Some(channels), _init: Some(init), events } } -fn reset(fail: u32) { FAIL.set(fail); OWNER_DROPS.set(0); BUFFER_DROPS.set(0); } +fn reset(fail: u32) { + FAIL.set(fail); OWNER_DROPS.set(0); BUFFER_DROPS.set(0); PMP_CALLS.set(0); +} fn retained(label: &str) { assert_eq!(OWNER_DROPS.get(), 0, "{label}: live session owner dropped"); assert_eq!(BUFFER_DROPS.get(), 0, "{label}: firmware buffer dropped"); @@ -221,17 +237,32 @@ def item(text, marker): std::mem::forget(shared); } } - reset(0); + for orphan_oom in [false, true] { + reset((1 << 13) | if orphan_oom { 1 << 7 } else { 0 }); + let shared = Shared::new(); + let result = Firmware::start_backend(&AsahiDevice, &PMP_CONFIG, booted(), &shared); + assert!(matches!(result, Err(-113)), "PMP readiness error changed"); + assert_eq!(PMP_CALLS.get(), 1, "PMP readiness was not reported"); + retained("PMP readiness failure"); + assert_eq!(shared.orphan.lock().is_some(), !orphan_oom); + std::mem::forget(shared); + } + for cfg in [&CONFIG, &PMP_CONFIG] { + // A pending PMP error must be irrelevant on a platform without PMP. + reset(if cfg.platform == hw::Platform::Adt { 1 << 13 } else { 0 }); let shared = Shared::new(); let booted = boot::start(&shared).expect("normal boot"); assert_eq!(booted.max_frequency_khz, 1234); let uat: Arc = booted.session.uat(); - let firmware = Firmware::start_backend(&AsahiDevice, &CONFIG, booted, &shared).expect("normal backend"); + let firmware = Firmware::start_backend(&AsahiDevice, cfg, booted, &shared).expect("normal backend"); + assert_eq!(PMP_CALLS.get(), u32::from(cfg.platform == hw::Platform::AdtWithPmp), + "PMP readiness used on the wrong platform"); assert!(Arc::ptr_eq(&uat, &firmware.session.uat())); assert!(!shared.orphaned.load(Ordering::Acquire)); retained("normal success"); std::mem::forget((shared, firmware)); - println!("PASS 23 retention cases: orphan OOM, six backend failure sites, four boot-completion sites and normal success"); + } + println!("PASS 26 retention cases: orphan OOM, six backend failure sites, four boot-completion sites, PMP readiness failure and both platform success paths"); println!("normal success owner_drops={} buffer_drops={}", OWNER_DROPS.get(), BUFFER_DROPS.get()); } ''' @@ -272,11 +303,16 @@ def main(): cwd=ROOT, text=True)) variants["consume-orphan-before-allocation"] = source.replace(orphan, item(old, " pub(crate) fn orphan(")) variants["unguarded-registry-allocation"] = source.replace(backend, item(old, " fn start_backend(")) + variants["ignore-pmp-readiness-error"] = source.replace( + "let ret = unsafe { boot::glue::asahi_g15_pmp_ready() };", + "let _ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; let ret = 0;") + variants["pmp-on-other-platform"] = source.replace( + "if cfg.platform != hw::Platform::AdtWithPmp {", "if false {") receipt = {"scope": __doc__, "sources": { name: hashlib.sha256(text.encode()).hexdigest() for name, text in [("mod.rs", mod), ("boot.rs", boot), ("kbox.rs", kbox)]}, "rustc": subprocess.check_output([os.environ.get("HOSTRUSTC", "rustc"), "--version"], text=True).strip(), - "cases": 23, "public_old_source_control": CONTROL_REF, "variants": {}} + "cases": 26, "public_old_source_control": CONTROL_REF, "variants": {}} if args.mutations: receipt["old_control_sha256"] = hashlib.sha256(old.encode()).hexdigest() for name, text in variants.items(): @@ -295,14 +331,20 @@ def main(): raise RuntimeError(result.stderr) else: expected = {"consume-orphan-before-allocation": "orphan allocation: live session owner dropped", - "unguarded-registry-allocation": "backend failure: live session owner dropped"}[name] - if result.returncode == 0 or expected not in result.stderr or "left: 5" not in result.stderr: + "unguarded-registry-allocation": "backend failure: live session owner dropped", + "ignore-pmp-readiness-error": "PMP readiness error changed", + "pmp-on-other-platform": "normal backend: -113"}[name] + if result.returncode == 0 or expected not in result.stderr: raise RuntimeError("INVALID mutant outcome " + name + "\n" + result.stderr) - receipt["variants"][name]["intended_owner_drops"] = 5 + if name in ("consume-orphan-before-allocation", "unguarded-registry-allocation"): + if "left: 5" not in result.stderr: + raise RuntimeError("INVALID owner-drop count " + name) + receipt["variants"][name]["intended_owner_drops"] = 5 + receipt["variants"][name]["intended_assertion"] = expected after = subprocess.run([str(out / "actual")], capture_output=True, text=True, check=True) receipt["actual_after_mutants"] = {"exit": after.returncode, "stdout": after.stdout, "stderr": after.stderr} receipt["fixed_invariants"] = {"live_session_owner_drops": 0, "firmware_buffer_drops": 0, - "checked_cases": 23} + "checked_cases": 26} (out / "result.json").write_text(json.dumps(receipt, indent=2) + "\n") print(json.dumps(receipt, indent=2)) diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index 5b728e95fc03e..c6e32d76cb415 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -1,12 +1,27 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Test the actual J516C record builder's narrow zero-word exemption. +"""Test the actual J516C record builder's narrow zero-word exemption and its word sources. Uses synthetic public data and temporary host copies of production Rust modules. No GPU, firmware file, private fixture, kernel build or hardware access. The golden T6030 inputs below are from SiliconForest 9f09c85f8d3b, except the full -hw.rs file refreshed at 709f0f08316a for the reviewed shared PB allocator changes. -Its T6030 configuration module is independently pinned to the original bytes. +hw.rs file refreshed at 709f0f08316a for the reviewed shared PB allocator changes +and again when the initial PB posting became a module parameter (the tuning +constant PB_BLOCKS removed), and when the T6031 board source became the +device tree with reference words instead of a board file (the Board enum; +the T6030 tables are unchanged), and for publication-bound slots (removal of +MAX_VMS and its lifetime-slot comment only), and when Board::DeviceTree was +gated on CONFIG_DRM_ASAHI_G15 for the T8122 runtime without the G15 backend +(one #[cfg] line; the tables are unchanged), and when the initial PB posting +and the idle reclaim became fixed tuning constants instead of module +parameters (four constants; the tables are unchanged), and when the two stamp +renumbering margins became tuning constants (the tables are unchanged). Its T6030 +configuration module is +independently pinned to the original bytes, +except for the J514S B0 FirmwareBuild row added to its firmware table: the +module is the original pinned bytes (0079ae08) plus that one 15-line row, and +the whole hw.rs pin was refreshed with it. prepare_board() is pinned too; +it was refreshed for the same Board change (its T6030 arm is unchanged). """ import argparse @@ -22,13 +37,13 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "e4619590e670ed88d2a9bb76b03853081c615d3d0ad1c6a68321ce8e615ef8a0", + "hw.rs": "c54a4f8342fb466f82f4b9cfed6a137ee1ca1f4d80ede4e1b7f397d39223558e", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { - ("hw.rs", "pub(crate) mod t6030 {"): "0079ae08fbe0e51cbf14e48711981a81048d1a1d3fc5495ab8d5c376283a760c", + ("hw.rs", "pub(crate) mod t6030 {"): "cab592b5d776be9ede93b96b1b0526c1d838ef2948a5e4a86caa4ec71f71c2ca", ("initdata.rs", "fn templates("): "c3115c9926d3d7d93317cb8510e93b04f30b0c256811d7137568e822dd93b8ee", - ("initdata.rs", "pub(crate) fn prepare_board("): "c59f968f125397efc63f75a49d3a8543c15c1fd4d50362f15683563398e64bc6", + ("initdata.rs", "pub(crate) fn prepare_board("): "ebef64ab87ef7b1f994eb7fa2a2424d05bc4fc9e88cbfbf18e51570bdfc4222d", ("board.rs", "pub(crate) fn from_adt("): "cecd0b28db5bd94d3b75c1bbb5f35204e8bf0699f74e5b3d08228c59025f6ac2", } @@ -49,7 +64,7 @@ def function(source, marker): use super::*; // Frozen pre-5131caff FROM_FILE offsets from SiliconForest 9f09c85f8d3b. - // This list is deliberately independent of both new lists. + // This list is deliberately independent of the three new lists. const OLD: [usize; 74] = [ 224,232,244,248,252,256,260,264,268,728,732,736,740,744,748, 992,1000,1012,1016,1024,1036,1496,1500,1504,1508,1512,1516, @@ -60,17 +75,32 @@ def function(source, marker): 45908,45912,45932,45940,45948,45956, ]; + const REF: Reference = Reference { + max_power_mw: 54321, + relative_power: [0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007], + fast_die_mask: 0x2111_2222_2333_2444, + alarm_mask: 0x2555_2666_2777_2888, + }; + + fn rows() -> [[u8; 12]; SENSOR_ROW_COUNT] { + [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"].map(|k| { + let mut r = [0x24u8; 12]; + r[..4].copy_from_slice(k); + r + }) + } + + fn row_words() -> Vec { + (SENSOR_ROWS..SENSOR_ROWS + 12 * SENSOR_ROW_COUNT).step_by(4).collect() + } + fn synthetic() -> (Vec, Vec, [Opp; 1], [u32; PROPS.len()]) { let mut file = vec![0u8; SIZE]; for (n, &at) in OLD.iter().enumerate() { put32(&mut file, at, 0x13570000 | (n as u32 + 1)).unwrap(); } - put32(&mut file, MAX_POWER, 54321).unwrap(); - for (n, key) in [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"].iter().enumerate() { - put32(&mut file, SENSOR_ROWS + 12*n, u32::from_le_bytes(**key)).unwrap(); - } // Public, synthetic OPPs: state0 off; eight voltage groups, five paired. - let mut main = vec![Opp {mhz:0, mv:[125;4], cells:4, mw:0}]; + let mut main = vec![Opp {mhz:0, mv:[125;4], cells:4}]; for (mv, freqs) in [ (600, &[300u32][..]), (650, &[500][..]), (700, &[700][..]), (750, &[800,900][..]), (800, &[950,1000][..]), @@ -78,10 +108,10 @@ def function(source, marker): (950, &[1250,1300][..]), ] { for &mhz in freqs { - main.push(Opp {mhz, mv:[mv;4], cells:4, mw:mhz*40}); + main.push(Opp {mhz, mv:[mv;4], cells:4}); } } - let cs = [Opp {mhz:24, mv:[635,0,0,0], cells:1, mw:0}]; + let cs = [Opp {mhz:24, mv:[635,0,0,0], cells:1}]; let mut props = [1u32; PROPS.len()]; props[PERIOD_MS] = 8; props[AVG_TC_MS] = 32; @@ -98,14 +128,16 @@ def function(source, marker): fn inputs<'a>(main: &'a [Opp], cs: &'a [Opp], props: [u32; PROPS.len()]) -> Inputs<'a> { Inputs {props, zones:&[], main, cs, afr:cs, slots:&[], empty_writable:&[], ttbat:0x100000, usc_base:0x1000000000, user_end:0x2000000000, - timestamps:0xfffffc2ffc000000, max_power_mw:54321} + timestamps:0xfffffc2ffc000000, sensor_rows: rows(), reference: &REF} } #[test] fn old74_is_exact_disjoint_partition() { - assert_eq!(FROM_FILE.len(), 32); + assert_eq!(REFERENCE_WORDS.len(), 14); + assert_eq!(row_words().len(), 18); assert_eq!(UNNEEDED.len(), 42); - let mut union = FROM_FILE.to_vec(); + let mut union = REFERENCE_WORDS.to_vec(); + union.extend(row_words()); union.extend_from_slice(&UNNEEDED); union.sort_unstable(); assert!(union.windows(2).all(|w| w[0] < w[1]), "duplicate or overlapping offsets"); @@ -115,30 +147,30 @@ def function(source, marker): } #[test] - fn actual_from_file_copies_only_retained32() { - let (file, _, _, _) = synthetic(); - let mut out = vec![0u8; SIZE]; - from_file(&mut out, &file).unwrap(); - for at in (0..SIZE).step_by(4) { - let expected = if FROM_FILE.contains(&at) {get32(&file, at).unwrap()} else {0}; - assert_eq!(get32(&out, at).unwrap(), expected, "copy offset +{at:#x}"); - } - } - - #[test] - fn actual_builder_leaves42_zero_and_preserves32() { + fn actual_builder_leaves42_zero_and_writes_the_32_from_their_sources() { let (file, main, cs, props) = synthetic(); - let mut out = vec![0u8; SIZE]; - from_file(&mut out, &file).unwrap(); + // Whatever the buffer held is not carried into the record. + let mut out = file.clone(); build(&mut out, &inputs(&main, &cs, props)).unwrap(); for &at in UNNEEDED.iter() { assert_ne!(get32(&file, at).unwrap(), 0, "fixture must exercise a real difference"); assert_eq!(get32(&out, at).unwrap(), 0, "built offset +{at:#x}"); } - for &at in FROM_FILE.iter() { - assert_eq!(get32(&out, at).unwrap(), get32(&file, at).unwrap(), "retained +{at:#x}"); + let mut reference = Vec::new(); + for r in REF.relative_power { reference.push(r); } + reference.push(REF.max_power_mw); + for m in [REF.fast_die_mask, REF.fast_die_mask, REF.alarm_mask] { + reference.push(m as u32); + reference.push((m >> 32) as u32); + } + for (&at, &want) in REFERENCE_WORDS.iter().zip(reference.iter()) { + assert_eq!(get32(&out, at).unwrap(), want, "reference +{at:#x}"); + } + let flat: Vec = rows().concat(); + for (k, &at) in row_words().iter().enumerate() { + assert_eq!(&out[at..at + 4], &flat[4 * k..4 * k + 4], "sensor row +{at:#x}"); } - // Model a matching board file with nonzero legacy values only at the42. + // Model a matching machine record with nonzero legacy values only at the42. let mut matching = out.clone(); for &at in UNNEEDED.iter() { put32(&mut matching, at, get32(&file, at).unwrap()).unwrap(); @@ -168,7 +200,7 @@ def function(source, marker): MUTANTS = { "broad-cross-check-exemption": ("cross_check", "UNNEEDED.contains(&at)", "true"), "nonzero-built-exempted": ("cross_check", "b == 0 && UNNEEDED.contains(&at)", "UNNEEDED.contains(&at)"), - "legacy-copy-reintroduced": ("from_file", "FROM_FILE.iter()", "FROM_FILE.iter().chain(UNNEEDED.iter())"), + "stale-buffer-kept": ("build", "out.fill(0);", ""), } @@ -246,7 +278,8 @@ def run(name, record): mutant = run(name, changed) if not mutant.returncode: raise RuntimeError("mutant survived: " + name) - regression = ("actual_from_file_copies_only_retained32" if func == "from_file" + regression = ("actual_builder_leaves42_zero_and_writes_the_32_from_their_sources" + if func == "build" else "actual_cross_check_exempts_only_zero_at_exact42") if regression + " ... FAILED" not in mutant.stdout: raise RuntimeError("new regression did not catch mutant: " + name) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 318f9e892505c..a075e2a1dab20 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -14,6 +14,9 @@ Negative controls read older revisions with `git show`, so the checkout needs the commits (a full or treeless clone, not a depth-1 one). +A test that reads device trees gets '{dtc}', this tree's scripts/dtc/dtc, built +with the kconfig tool unless --dtc names one. + Usage: tools/asahi/run-host-tests.py [--jobs N] [--work DIR] [--only SUBSTR] """ import argparse @@ -61,6 +64,7 @@ def mutations(flag, names, base, expect=KILL): 'apple-pwm-pm-tests.py': [([], PASS), (['--negative-control'], PASS)], 'apple-dart-fw-tests.py': [(out(), PASS)], 'apple-dart-range-tests.py': [([], PASS)], + 'asahi-stats-raw-tests.py': [([], PASS)], 'atc-usb3-tests.py': [(out(), PASS), (out() + ['--mutations'], PASS)], 'avd-av1-tail-tests.py': [(out(), PASS)] + [(out() + ['--case', c], PASS) for c in ('copy', 'seed', 'aligned')] @@ -89,6 +93,7 @@ def mutations(flag, names, base, expect=KILL): 'display26-tests.py': [([], PASS)], # The old revision publishes the fd before copy_to_user() succeeds. 'dma-heap-fd-tests.py': [(out() + ['--old-ref', 'cb732d027aa18e1fcf9d2797f47d20b179ebc59c'], PASS)], + 'g15-display-order-tests.py': [([], PASS)], 'g15-input-error-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS), (out('--output') + ['--ref', 'ede99b3c1990b4a4a8c045fdf627a2bd0f14446a'], PASS)], 'g15-maintained-handoff-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], @@ -136,6 +141,7 @@ def mutations(flag, names, base, expect=KILL): 't6030-clock-tests.py': [(out(), PASS)], 't6030-display-admission-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 't6030-display-linkage-tests.py': [(out('--output') + ['--conf', '{conf}'], PASS)], + 't6030-pmp-dt-tests.py': [(out('--output') + ['--dtc', '{dtc}'], PASS)], 't6030-pmp-graph-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 't6030-pmp-supplier-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 't6030-wiring-tests.py': [(out(), PASS), (out() + ['--old-classifier', 'tb'], PASS), @@ -168,7 +174,7 @@ def mutations(flag, names, base, expect=KILL): ] -def jobs(work, conf): +def jobs(work, conf, dtc): found = {p.name for p in ASAHI.iterdir() if re.search(r'-tests\.(py|rs|sh)$', p.name) and p.name != Path(__file__).name} known = set(ASAHI_TESTS) | set(ASAHI_RUST) | set(ASAHI_SHELL) @@ -190,9 +196,11 @@ def jobs(work, conf): for name, runs in ASAHI_TESTS.items(): for index, (args, expect) in enumerate(runs): label = name.removesuffix('.py') + ('' if not args or args == out() or args == out('--output') - else ' ' + ' '.join(a for a in args if a not in ('{out}', '--out', '--output', '{conf}', '--conf'))) + else ' ' + ' '.join(a for a in args if a not in ('{out}', '--out', '--output', '{conf}', '--conf', + '{dtc}', '--dtc'))) d = work / f'{name}.{index}' - cmd = ['python3', '-I', '-B', str(ASAHI / name)] + [a.replace('{out}', str(d / 'out')).replace('{conf}', str(conf)) for a in args] + cmd = ['python3', '-I', '-B', str(ASAHI / name)] + [a.replace('{out}', str(d / 'out')).replace('{conf}', str(conf)) + .replace('{dtc}', str(dtc)) for a in args] result.append((label, cmd, ASAHI, expect, d, {})) for name in ASAHI_RUST: d = work / name @@ -233,6 +241,7 @@ def main(): ap.add_argument('--jobs', type=int, default=os.cpu_count() or 2) ap.add_argument('--work', type=Path, help='keep logs and build products here (default: a temporary directory)') ap.add_argument('--conf', type=Path, help='a built scripts/kconfig/conf (default: build one)') + ap.add_argument('--dtc', type=Path, help='a built scripts/dtc/dtc (default: build one)') ap.add_argument('--only', help='run only jobs whose name contains this text') ap.add_argument('--timeout', type=int, default=900) args = ap.parse_args() @@ -240,13 +249,18 @@ def main(): work = args.work or Path(tempfile.mkdtemp(prefix='aurora-host-tests-')) work = work.resolve() work.mkdir(parents=True, exist_ok=True) - conf = args.conf - if conf is None: - build = work / 'kconfig' + conf, dtc = args.conf, args.dtc + build = work / 'kconfig' + if conf is None or dtc is None: subprocess.run(['make', '-s', '-C', str(ROOT), f'O={build}', 'ARCH=arm64', 'allnoconfig'], check=True, stdout=subprocess.DEVNULL) + if conf is None: conf = build / 'scripts/kconfig/conf' - todo = jobs(work, conf.resolve()) + if dtc is None: + subprocess.run(['make', '-s', '-C', str(ROOT), f'O={build}', 'ARCH=arm64', 'scripts_dtc'], + check=True, stdout=subprocess.DEVNULL) + dtc = build / 'scripts/dtc/dtc' + todo = jobs(work, conf.resolve(), dtc.resolve()) if args.only: todo = [j for j in todo if args.only in j[0]] print(f'{len(todo)} runs, {args.jobs} at a time, logs in {work}', flush=True) diff --git a/tools/asahi/t6030-display-admission-tests.c b/tools/asahi/t6030-display-admission-tests.c index e2c75eddb5ea5..ebf3e2e868324 100644 --- a/tools/asahi/t6030-display-admission-tests.c +++ b/tools/asahi/t6030-display-admission-tests.c @@ -22,12 +22,14 @@ struct device { struct device_link { int unused; }; -static pthread_mutex_t apple_t6030_display_lock = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t apple_g15_display_lock = PTHREAD_MUTEX_INITIALIZER; static struct device *apple_t6030_display; +static struct device *apple_t6031_display; static struct device_node display_node = { "apple,t6030-display-subsystem" }, gpu_node = { "apple,agx-t6030" }, - other_node = { "apple,t6031-display-subsystem" }; -static struct device display, duplicate, gpu; + other_node = { "apple,display-subsystem" }, + t6031_display_node = { "apple,t6031-display-subsystem" }; +static struct device display, duplicate, gpu, t6031_display; static const char *machine = "apple,t6030"; static int pmp_calls, link_calls, pmp_result; static bool link_ok = true; @@ -106,7 +108,7 @@ static int apple_pmp_link_device(struct device *dev) #include "production.h" static void reset(void) { - assert(!apple_t6030_display && !locked); + assert(!apple_t6030_display && !apple_t6031_display && !locked); display = (struct device){ .of_node = &display_node, .refs = 1 }; duplicate = (struct device){ .of_node = &display_node, .refs = 1 }; gpu = (struct device){ .of_node = &gpu_node, .refs = 1 }; @@ -127,7 +129,7 @@ static void *admit_thread(void *unused) static void *withdraw_thread(void *unused) { (void)unused; - apple_t6030_display_withdraw(&display); + apple_g15_display_withdraw(&display); atomic_store(&withdrew, true); return NULL; } @@ -140,18 +142,18 @@ int main(void) assert(asahi_g15_pmp_prepare(&gpu) == -EPROBE_DEFER && !pmp_calls); cases++; /* Component platform bind alone never calls this publication. Retry succeeds only afterward. */ - assert(apple_t6030_display_publish(&display) == 0 && + assert(apple_g15_display_publish(&display) == 0 && atomic_load(&display.refs) == 2); cases++; assert(asahi_g15_pmp_prepare(&gpu) == 0 && pmp_calls == 1); cases++; - assert(apple_t6030_display_publish(&display) == 0 && + assert(apple_g15_display_publish(&display) == 0 && atomic_load(&display.refs) == 2); cases++; - assert(apple_t6030_display_publish(&duplicate) == -EBUSY && + assert(apple_g15_display_publish(&duplicate) == -EBUSY && apple_t6030_display == &display); cases++; - apple_t6030_display_withdraw(&duplicate); + apple_g15_display_withdraw(&duplicate); assert(apple_t6030_display == &display && atomic_load(&display.refs) == 2); cases++; @@ -170,17 +172,27 @@ int main(void) assert(apple_drm_t6030_link_device(&gpu) == -ENODEV); cases++; gpu.of_node = &gpu_node; - apple_t6030_display_withdraw(&display); + apple_g15_display_withdraw(&display); assert(atomic_load(&display.refs) == 1 && !apple_t6030_display); cases++; assert(asahi_g15_pmp_prepare(&gpu) == -EPROBE_DEFER && pmp_calls == 2); cases++; display.of_node = &other_node; - assert(apple_t6030_display_publish(&display) == 0 && - !apple_t6030_display && atomic_load(&display.refs) == 1); + assert(apple_g15_display_publish(&display) == 0 && + !apple_t6030_display && !apple_t6031_display && + atomic_load(&display.refs) == 1); + cases++; + /* The T6031 display has its own slot: it never satisfies T6030 admission. */ + t6031_display = (struct device){ .of_node = &t6031_display_node, .refs = 1 }; + assert(apple_g15_display_publish(&t6031_display) == 0 && + apple_t6031_display == &t6031_display && !apple_t6030_display && + atomic_load(&t6031_display.refs) == 2); + assert(asahi_g15_pmp_prepare(&gpu) == -EPROBE_DEFER && pmp_calls == 2); + apple_g15_display_withdraw(&t6031_display); + assert(!apple_t6031_display && atomic_load(&t6031_display.refs) == 1); cases++; reset(); - assert(!apple_t6030_display_publish(&display)); + assert(!apple_g15_display_publish(&display)); atomic_store(&pause_link, true); pthread_t admit, withdraw; assert(!pthread_create(&admit, NULL, admit_thread, NULL)); diff --git a/tools/asahi/t6030-display-admission-tests.py b/tools/asahi/t6030-display-admission-tests.py index a1da987377f18..479c11b00fed0 100644 --- a/tools/asahi/t6030-display-admission-tests.py +++ b/tools/asahi/t6030-display-admission-tests.py @@ -2,9 +2,11 @@ # SPDX-License-Identifier: GPL-2.0-only """Actual T6030 display publisher and G15 admission, with pthread boundaries. -Tests initial admission and pointer lifetime, not safe shutdown of retained G15 -firmware after forced display unbind. Full DRM publication placement is checked -as source ordering separately from executable host boundary controls. +The publisher is the shared G15 display slot code; T6030 admission goes through +the shared link helper. Tests initial admission and pointer lifetime, not safe +shutdown of retained G15 firmware after forced display unbind. Full DRM +publication placement is checked as source ordering separately from executable +host boundary controls. """ import argparse import hashlib @@ -26,23 +28,26 @@ def main(): glue = (ROOT / 'drivers/gpu/drm/asahi/g15/glue.c').read_text() init = extract.function(drv, 'static int apple_drm_init(') uninit = extract.function(drv, 'static void apple_drm_uninit(') - assert init.index('ret = drm_dev_register(') < init.index('drm_client_setup_with_fourcc(') < init.index('ret = apple_t6030_display_publish(') - assert uninit.index('apple_t6030_display_withdraw(') < uninit.index('apple_drm_quiesce_connectors(') < uninit.index('drm_dev_unplug(') + assert init.index('ret = drm_dev_register(') < init.index('drm_client_setup_with_fourcc(') < init.index('ret = apple_g15_display_publish(') + assert uninit.index('apple_g15_display_withdraw(') < uninit.index('apple_drm_quiesce_connectors(') < uninit.index('drm_dev_unplug(') assert 'drm_dev_unregister(&apple->drm);\n\t\tgoto err_unbind;' in init - code = '\n'.join([extract.function(drv, 'static int apple_t6030_display_publish('), - extract.function(drv, 'static void apple_t6030_display_withdraw('), + code = '\n'.join([extract.function(drv, 'static struct device **apple_g15_display_slot('), + extract.function(drv, 'static int apple_g15_display_publish('), + extract.function(drv, 'static void apple_g15_display_withdraw('), + extract.function(drv, 'static int apple_g15_link_device('), extract.function(drv, 'int apple_drm_t6030_link_device('), extract.function(glue, 'int asahi_g15_pmp_prepare(struct device *dev)\n')]) variants = [('control', code)] if args.mutations: for name, old, new in [ - ('no-ready-gate', 'if (!apple_t6030_display)', 'if (false)'), - ('no-display-get', 'apple_t6030_display = get_device(dev);', 'apple_t6030_display = dev;'), + ('no-ready-gate', 'if (!*display)', 'if (false)'), + ('no-display-get', '*slot = get_device(dev);', '*slot = dev;'), ('no-display-put', 'put_device(old);', '(void)old;'), - ('replace-live-provider', 'return apple_t6030_display == dev ? 0 : -EBUSY;', 'apple_t6030_display = dev;'), + ('replace-live-provider', 'return *slot == dev ? 0 : -EBUSY;', '*slot = dev;'), ('ignore-display-error', 'if (ret)\n\t\treturn ret;', 'if (false)\n\t\treturn ret;'), - ('wrong-gpu', '!of_device_is_compatible(consumer->of_node, "apple,agx-t6030")', 'false'), - ('withdraw-wrong-provider', 'if (apple_t6030_display == dev)', 'if (((void)dev, true))'), + ('wrong-gpu', '!of_device_is_compatible(consumer->of_node, gpu)', '((void)gpu, false)'), + ('withdraw-wrong-provider', 'if (*slot == dev)', 'if (((void)dev, true))'), + ('shared-display-slot', 'return &apple_t6031_display;', 'return &apple_t6030_display;'), ]: assert code.count(old) == 1, (name, code.count(old)) variants.append((name, code.replace(old, new))) diff --git a/tools/asahi/t6030-pmp-dt-tests.py b/tools/asahi/t6030-pmp-dt-tests.py new file mode 100644 index 0000000000000..913e8d31aa7d7 --- /dev/null +++ b/tools/asahi/t6030-pmp-dt-tests.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Check static PMP ownership in the compiled T6030 device trees.""" + +import argparse +from pathlib import Path +import subprocess + + +ROOT = Path(__file__).resolve().parents[2] +BOARDS = ("t6030-j514s", "t6030-j516s") +PMP_NODES = { + "iommu@350300000", + "pmp_report@3503c0000", + "pmp@350500000", + "mbox@350c08000", +} +PMP_COMPATIBLES = {"apple,t6000-pmp-v2", "apple,t6030-pmp-v2-report"} + + +def check_default_compatibles(dtb, board): + pending = ["/"] + while pending: + node = pending.pop() + raw = subprocess.check_output( + ["fdtget", "-t", "bx", "-d", "", str(dtb), node, "compatible"], + text=True, + ) + # Match the gate's whole-tree OF lookup, including disabled nodes + # and every string in a compatible list. + compatible = bytes(int(value, 16) for value in raw.split()) + matches = PMP_COMPATIBLES.intersection( + value.decode("ascii").lower() for value in compatible.split(b"\0") + if value + ) + if matches: + raise AssertionError(f"{board} default: static PMP compatible " + f"{sorted(matches)} at {node}") + children = subprocess.check_output( + ["fdtget", "-l", str(dtb), node], text=True, + ) + pending.extend(node.rstrip("/") + "/" + child + for child in children.splitlines()) + + +def check_profile(tree, output, dtc, board, static): + profile = "static" if static else "default" + dtb = output / f"{board}-{profile}.dtb" + command = [ + "cpp", "-nostdinc", "-undef", "-D__DTS__", + "-I", str(tree / "scripts/dtc/include-prefixes"), + "-x", "assembler-with-cpp", + ] + if static: + command.append("-DAPPLE_USE_PMP") + command.append(str(tree / f"arch/arm64/boot/dts/apple/{board}.dts")) + source = subprocess.check_output(command) + subprocess.run([dtc, "-q", "-I", "dts", "-O", "dtb", "-o", str(dtb)], + input=source, check=True) + + children = subprocess.check_output(["fdtget", "-l", str(dtb), "/soc"], text=True) + present = PMP_NODES.intersection(children.split()) + expected = PMP_NODES if static else set() + if present != expected: + raise AssertionError(f"{board} {profile}: static PMP nodes " + f"{sorted(present)}, expected {sorted(expected)}") + if not static: + check_default_compatibles(dtb, board) + + alias = subprocess.run(["fdtget", "-t", "s", str(dtb), "/aliases", "pmp"], + capture_output=True, text=True) + if static: + if alias.returncode or alias.stdout.strip() != "/soc/pmp@350500000": + raise AssertionError(f"{board} {profile}: missing or incorrect PMP alias") + elif alias.returncode == 0: + raise AssertionError(f"{board} {profile}: unexpected static PMP alias") + print(f"{board} {profile}: PASS") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tree", type=Path, default=ROOT) + parser.add_argument("--dtc", required=True, help="this kernel's built scripts/dtc/dtc") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + for board in BOARDS: + for static in (False, True): + check_profile(args.tree.resolve(), args.output, args.dtc, board, static) + + +if __name__ == "__main__": + main() diff --git a/tools/asahi/t6030-pmp-supplier-tests.c b/tools/asahi/t6030-pmp-supplier-tests.c index b581f3863a232..ecb6dcec9fff5 100644 --- a/tools/asahi/t6030-pmp-supplier-tests.c +++ b/tools/asahi/t6030-pmp-supplier-tests.c @@ -130,6 +130,8 @@ static int pmp_lock, ready_calls, add_calls, add_result; static int poll_calls, device_locked, power_calls, power_result, link_calls; static int find_calls, lockable, present_device, link_ok; static int case_count; +/* The report transaction itself is covered by the G15 PMP-vote selftest. */ +static int gfx_check_result, gfx_check_calls, gfx_set_calls; static struct device_node *of_node_get(struct device_node *node) { @@ -375,6 +377,19 @@ static void queue_work(void *wq, int *work) managed_start.result = ready_result; managed_start.started.done = true; } +static int apple_pmp_report_check_t6030_gfx(struct device *dev) +{ + assert(dev == pmp_device && pmp_lock == 1); + gfx_check_calls++; + return gfx_check_result; +} +static int apple_pmp_report_set_t6030_gfx(struct device *dev, bool enabled) +{ + assert(dev == pmp_device && pmp_lock == 1); + (void)enabled; + gfx_set_calls++; + return 0; +} #include "t6030-pmp-production.h" static void balanced(void) @@ -457,6 +472,7 @@ static void reset(void) pmp_data = (void *)1; ready_calls = add_calls = add_result = poll_calls = 0; power_calls = power_result = link_calls = find_calls = 0; + gfx_check_result = gfx_check_calls = gfx_set_calls = 0; lockable = present_device = link_ok = 1; status_word = PMP_REPORT_READY; } @@ -498,6 +514,9 @@ int main(void) balanced(); case_count++; RUN((void)0, 0, 0, 1); + assert(gfx_check_calls == 1 && gfx_set_calls == 0); + RUN(gfx_check_result = -EINVAL, -EINVAL, 0, 0); + assert(gfx_check_calls == 1 && gfx_set_calls == 0); RUN(report14.child = &entry7; entry7.sibling = &entry17; entry17.sibling = NULL, 0, 0, 1); RUN(apple_pmp_starts.first = NULL, -ENODEV, 0, 0); diff --git a/tools/asahi/t6030-pmp-supplier-tests.py b/tools/asahi/t6030-pmp-supplier-tests.py index d323dffbef728..269006b36aa4f 100755 --- a/tools/asahi/t6030-pmp-supplier-tests.py +++ b/tools/asahi/t6030-pmp-supplier-tests.py @@ -6,7 +6,9 @@ extracted. OF enumeration/property storage, device binding, MMIO polling, and locks are controlled host boundaries. The T6030 managed entry wait and start wait are extracted too; the firmware/workqueue boundary is controlled. -This models ownership/error handling, not real teardown concurrency or DMA. +The GFX-report service is modeled separately from supplier readiness; its +transaction has its own production-body PMP-vote selftest. This models +ownership/error handling, not real teardown concurrency or DMA. """ import argparse import hashlib @@ -139,6 +141,8 @@ def main(): ("gate-config", "IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) && gate_soc == &gate_t6030", "gate_soc == &gate_t6030"), ("gate-phandle-shape", 'of_property_count_u32_elems(gpu, "apple,pmp") != 1', "false"), ("serialized-command", "\tguard(mutex)(&pmp_lock);", "\t(void)&pmp_lock;"), + ("gfx-check-error", "ret = apple_pmp_report_check_t6030_gfx(pmp_device);", + "ret = (apple_pmp_report_check_t6030_gfx(pmp_device), 0);"), ] for name, old, new in mutations: assert code.count(old) == 1, (name, code.count(old)) diff --git a/tools/asahi/tbnet-refill-fixture.h b/tools/asahi/tbnet-refill-fixture.h index e88dfd62831db..d9dee1baab9b2 100644 --- a/tools/asahi/tbnet-refill-fixture.h +++ b/tools/asahi/tbnet-refill-fixture.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #define TBNET_RING_SIZE 256 #define TBNET_FRAME_SIZE 4096 @@ -92,7 +93,17 @@ static bool in_softirq(void){return fake_softirq;} static int smp_processor_id(void){return 0;} static void schedule_work(struct work_struct *w){} static bool queue_delayed_work(int wq,struct delayed_work *w,unsigned long delay){assert(delay==100);pthread_mutex_lock(&gate);bool new=!w->pending;if(new){w->pending=true;retry_queues++;retry_delay=delay;}pthread_mutex_unlock(&gate);return new;} -static void cancel_delayed_work_sync(struct delayed_work *w){pthread_mutex_lock(&gate);cancel_waiting=true;while(w->running)pthread_cond_wait(&cv,&gate);w->pending=false;cancel_waiting=false;pthread_mutex_unlock(&gate);} +static void cancel_delayed_work_sync(struct delayed_work *w) +{ + pthread_mutex_lock(&gate); + cancel_waiting = true; + pthread_cond_broadcast(&cv); + while (w->running) + pthread_cond_wait(&cv, &gate); + w->pending = false; + cancel_waiting = false; + pthread_mutex_unlock(&gate); +} static void fire_retry(struct delayed_work *w){pthread_mutex_lock(&gate);assert(w->pending&&!w->running);w->pending=false;w->running=true;pthread_mutex_unlock(&gate);w->work.fn(&w->work);pthread_mutex_lock(&gate);w->running=false;pthread_cond_broadcast(&cv);pthread_mutex_unlock(&gate);} static void nhi_ring_interrupt_mask(struct tb_ring *r,bool mask){r->masked=mask;} diff --git a/tools/asahi/tbnet-refill-tests.py b/tools/asahi/tbnet-refill-tests.py index f5a6e98e64175..641f8758bc927 100644 --- a/tools/asahi/tbnet-refill-tests.py +++ b/tools/asahi/tbnet-refill-tests.py @@ -56,6 +56,18 @@ def extract(name,text=source): static void *setup_thread(void *p){pthread_mutex_lock(&gate);setup_active=true;pthread_mutex_unlock(&gate);connect(p);pthread_mutex_lock(&gate);setup_active=false;pthread_cond_broadcast(&cv);pthread_mutex_unlock(&gate);return NULL;} static void *tear_thread(void *p){tbnet_tear_down(p,false);return NULL;} static void wait_paused(void){pthread_mutex_lock(&gate);while(!paused)pthread_cond_wait(&cv,&gate);pthread_mutex_unlock(&gate);} +static void wait_cancel(void) +{ + struct timespec deadline; + + assert(!clock_gettime(CLOCK_REALTIME, &deadline)); + deadline.tv_sec += 2; + pthread_mutex_lock(&gate); + while (!cancel_waiting) + assert(!pthread_cond_timedwait(&cv, &gate, &deadline)); + assert(ring_stops == 0 && pool_destroy_calls == 0); + pthread_mutex_unlock(&gate); +} static void release_poll(void){pthread_mutex_lock(&gate);resume_poll=1;pthread_cond_broadcast(&cv);pthread_mutex_unlock(&gate);} int main(int argc,char **argv){assert(argc==2);char *c=argv[1];struct tbnet n;struct net_device d;struct tb tb={0};struct tb_xdomain x={&tb};struct tb_service s={0};struct device dev={&s}; if(!strcmp(c,"pool-create-failure")){pool_error=1;memset(&n,0,sizeof(n));memset(&d,0,sizeof(d));d.priv=&n;n.dev=&d;n.xd=&x;n.svc=&s;pthread_mutex_init(&n.connection_lock,NULL);assert(tbnet_open(&d)==-ENOMEM);assert(!n.rx_ring.pool&&!n.rx_ring.ring&&!n.tx_ring.ring&&ring_frees==2&&enable_calls==0);return 0;} @@ -80,7 +92,7 @@ def extract(name,text=source): if(!strcmp(c,"retry-stop")||!strcmp(c,"retry-disconnect")||!strcmp(c,"retry-suspend")||!strcmp(c,"retry-shutdown")){exhaust(&n);int a=alloc_calls;if(!strcmp(c,"retry-stop"))tbnet_stop(&d);else if(!strcmp(c,"retry-suspend"))tbnet_suspend(&dev);else if(!strcmp(c,"retry-shutdown"))tbnet_shutdown(&s);else tbnet_tear_down(&n,false);assert(!n.rx_refill_work.pending&&!n.napi.enabled&&alloc_calls==a);assert(!n.rx_ring.ring||!n.rx_ring.ring->running);return 0;} if(!strcmp(c,"retry-reconnect")){exhaust(&n);tbnet_tear_down(&n,false);assert(!n.rx_refill_work.pending);alloc_limit=-1;n.login_sent=n.login_received=true;n.remote_transmit_path=12;connect(&n);assert(n.napi.enabled&&n.rx_ring.prod==256&&!n.rx_refill_work.pending);tbnet_stop(&d);return 0;} if(!strcmp(c,"retry-completion-tail")){alloc_limit=0;complete_frames(&n,1,2);pause_at=2;pthread_t a,b;assert(!pthread_create(&a,NULL,poll_thread,&n));wait_paused();assert(n.rx_refill_work.pending);assert(!pthread_create(&b,NULL,stop_thread,&n));usleep(30000);assert(ring_stops==0&&n.rx_refill_work.pending);release_poll();assert(!pthread_join(a,NULL)&&!pthread_join(b,NULL));assert(!n.rx_refill_work.pending&&ring_stops==2);return 0;} - if(!strcmp(c,"retry-running-teardown")){exhaust(&n);pause_at=6;pthread_t a,b;assert(!pthread_create(&a,NULL,retry_thread,&n));wait_paused();assert(!pthread_create(&b,NULL,stop_thread,&n));usleep(30000);assert(ring_stops==0&&pool_destroy_calls==0&&cancel_waiting);release_poll();assert(!pthread_join(a,NULL)&&!pthread_join(b,NULL));assert(!n.rx_refill_work.pending&&!n.rx_refill_work.running&&!n.napi.owner&&ring_stops==2&&pool_destroy_calls==1);return 0;} + if(!strcmp(c,"retry-running-teardown")){exhaust(&n);pause_at=6;pthread_t a,b;assert(!pthread_create(&a,NULL,retry_thread,&n));wait_paused();assert(!pthread_create(&b,NULL,stop_thread,&n));wait_cancel();release_poll();assert(!pthread_join(a,NULL)&&!pthread_join(b,NULL));assert(!n.rx_refill_work.pending&&!n.rx_refill_work.running&&!n.napi.owner&&ring_stops==2&&pool_destroy_calls==1);return 0;} if(!strcmp(c,"retry-already-polling")){exhaust(&n);tbnet_start_poll(&n);fire_retry(&n.rx_refill_work);int rearm=rearm_calls;assert(n.napi.missed&&tbnet_poll(&n.napi,64)==0&&n.napi.owner&&n.rx_refill_work.pending&&rearm_calls==rearm);alloc_limit=-1;assert(tbnet_poll(&n.napi,64)==0&&!n.napi.owner&&n.rx_ring.prod-n.rx_ring.cons==256);return 0;} if(!strcmp(c,"retry-disabled-worker")){exhaust(&n);napi_disable(&n.napi);int a=alloc_calls,sch=schedule_calls;fire_retry(&n.rx_refill_work);assert(!n.napi.owner&&schedule_calls==sch&&alloc_calls==a);return 0;} if(!strcmp(c,"retry-counter-wrap")){n.rx_ring.cons=0xffffff00;n.rx_ring.prod=0;exhaust(&n);alloc_limit=-1;retry_poll(&n);assert(n.rx_ring.cons==0&&n.rx_ring.prod==256&&!n.rx_refill_work.pending);return 0;} diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c new file mode 100644 index 0000000000000..806148f64d1bf --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise the production PMP vote transaction with failed firmware replies. */ +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +#define CONFIG_APPLE_T6030_GPU_PMP_LINK 1 +#define IS_ENABLED(x) (x) +#define guard(kind) ignore_lock +#define dev_warn(...) ((void)0) +static int pmp_lock; +static void ignore_lock(void *p) { (void)p; } +struct device { void *of_node; }; +static struct device owner; +static struct device *pmp_device = &owner; +static const void *pmp_data = &owner; +static bool t6030, requested; +static int ready, check, power_result[2], report_result[3], powers, reports; +static char calls[16]; +static unsigned int used; +static void called(char c) { assert(used < sizeof(calls) - 1); calls[used++] = c; } +static bool of_machine_is_compatible(const char *s) { (void)s; return t6030; } +static bool of_device_is_compatible(void *p, const char *s) +{ + (void)p; (void)s; return t6030; +} +static int apple_pmp_report_wait_supplier_ready(struct device *d, unsigned int ms) +{ + assert(d == &owner && ms == 5000); called('W'); return ready; +} +static int apple_pmp_send_power_command(const void *p, u8 cmd, u16 id, u32 enabled) +{ + assert(p == &owner && cmd == 0x0f && id == 5); + called(enabled ? 'P' : 'p'); assert(powers < 2); return power_result[powers++]; +} +static int apple_pmp_report_check_t6030_gfx(struct device *d) +{ + assert(d == &owner); called('V'); return check; +} +static int apple_pmp_report_set_t6030_gfx(struct device *d, bool enabled) +{ + assert(d == &owner); called(enabled ? 'R' : 'r'); requested = enabled; + assert(reports < 3); return report_result[reports++]; +} +#include "pmp-vote-production.h" +static void reset(void) +{ + pmp_data = &owner; t6030 = true; requested = false; ready = check = 0; powers = reports = used = 0; + memset(calls, 0, sizeof(calls)); + memset(power_result, 0, sizeof(power_result)); + memset(report_result, 0, sizeof(report_result)); +} +static void result(int expected, const char *sequence, bool enabled) +{ + assert(apple_pmp_set_device_power(0x0f, 5, enabled) == expected); + assert(strcmp(calls, sequence) == 0); +} +int main(void) +{ + reset(); + assert(apple_pmp_set_device_power(0, 5, 1) == -EINVAL); + assert(apple_pmp_set_device_power(0x0f, 16, 1) == -EINVAL); + assert(apple_pmp_set_device_power(0x0f, 5, 2) == -EINVAL); + assert(!used); + pmp_data = NULL; result(-ENODEV, "", true); + reset(); ready = -EAGAIN; result(-EAGAIN, "W", true); + reset(); t6030 = false; result(0, "WP", true); + reset(); t6030 = false; result(0, "Wp", false); + reset(); check = -EINVAL; result(-EINVAL, "WV", true); + reset(); result(0, "WVP", true); + /* Power-on must not start bandwidth tracking before firmware READY. */ + assert(!reports); + assert(apple_pmp_t6030_gpu_ready() == 0); + assert(strcmp(calls, "WVPrR") == 0 && requested); + reset(); requested = true; + assert(apple_pmp_t6030_gpu_ready() == 0); + assert(strcmp(calls, "rR") == 0 && requested); + reset(); pmp_data = NULL; + assert(apple_pmp_t6030_gpu_ready() == -ENODEV && !used); + reset(); report_result[0] = -ETIMEDOUT; + assert(apple_pmp_t6030_gpu_ready() == -ETIMEDOUT); + assert(strcmp(calls, "r") == 0); + reset(); report_result[1] = -ETIMEDOUT; + assert(apple_pmp_t6030_gpu_ready() == -ETIMEDOUT); + assert(strcmp(calls, "rR") == 0); + reset(); result(0, "WVrp", false); + reset(); power_result[0] = -ETIMEDOUT; result(-ETIMEDOUT, "WVP", true); + reset(); report_result[0] = -ETIMEDOUT; result(-ETIMEDOUT, "WVrR", false); + reset(); report_result[0] = -ETIMEDOUT; report_result[1] = -EIO; + result(-ETIMEDOUT, "WVrR", false); + reset(); power_result[0] = -EIO; result(-EIO, "WVrpR", false); + puts("production PMP vote: scope, sequencing, reply errors and rollback: PASS"); +} diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index 35c96a7f36176..2ae5be4b57e56 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -102,6 +102,7 @@ pub type Result = std::result::Result; pub const EINVAL: Error = -22; pub const EFAULT: Error = -14; + pub const ENOSPC: Error = -28; pub use crate::{pr_warn, static_assert}; } #[macro_export] @@ -113,6 +114,7 @@ pub const UAT_PGSZ: usize = 0x4000; pub const UAT_PGMSK: usize = UAT_PGSZ - 1; pub struct Vm(pub Vec); + pub type VmReadGuard = Vm; impl Vm { pub fn read_user(&self, va: u64, out: &mut [u8]) -> Result { let start = usize::try_from(va).map_err(|_| EFAULT)?; @@ -124,8 +126,19 @@ } mod command { use crate::{mmu, prelude::*}; + use std::cell::Cell; pub const USC_WINDOW: u64 = 1 << 32; - pub fn cover(vm: &mmu::Vm, start: u64, end: u64, _: bool, _: bool) -> Result { + /// Addresses from here on are mapped and readable without backing in the mock VM. + pub const SYNTHETIC: u64 = 1 << 40; + thread_local! { + /// cover() calls: the page checks the walk made. + pub static COVERS: Cell = Cell::new(0); + } + pub fn cover(vm: &mut mmu::VmReadGuard, start: u64, end: u64, _: bool, _: bool) -> Result { + COVERS.with(|c| c.set(c.get() + 1)); + if start >= SYNTHETIC { + return Ok(()); + } let mut bytes = vec![0; usize::try_from(end - start).map_err(|_| EINVAL)?]; vm.read_user(start, &mut bytes) } @@ -144,28 +157,100 @@ } #[test] fn terminator_and_known_barrier() { - assert_eq!(gate::cdm(&vm(&[0x40000000]), 0, 0), Ok(())); - assert_eq!(gate::cdm(&vm(&[0x60000160, 0x40000000]), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&[0x40000000]), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&[0x60000160, 0x40000000]), 0, 0), Ok(())); } #[test] fn cyclic_links_are_rejected() { - assert_eq!(gate::cdm(&vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); } + /// Words of 4 bytes that fill the byte budget: the root block and one per link. + const BUDGET_WORDS: usize = 65 * 0x10000 / 4; #[test] - fn record_budget_requires_a_terminator() { - assert_eq!(gate::cdm(&vm(&vec![0x60000160; 8192]), 0, 0), Err(EINVAL)); + fn byte_budget_requires_a_terminator() { + assert_eq!(gate::cdm(&mut vm(&vec![0x60000160; BUDGET_WORDS + 1]), 0, 0), Err(EINVAL)); } #[test] fn final_record_may_terminate() { - let mut words = vec![0x60000160; 8192]; - words[8191] = 0x40000000; - assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + let mut words = vec![0x60000160; BUDGET_WORDS + 1]; + words[BUDGET_WORDS - 1] = 0x40000000; + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); + words[BUDGET_WORDS - 1] = 0x60000160; + words[BUDGET_WORDS] = 0x40000000; + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Err(EINVAL)); + } + #[test] + fn more_than_8192_records_are_admitted() { + // 1024 draws of ten records each, as a tessellated multi-draw writes them. + let mut words = vec![0x60000160; 10240]; + words.push(0x40000000); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); + } + #[test] + fn link_budget_still_bounds_the_walk() { + // 65 links, each to the next: the 65th is refused. + let mut words = Vec::new(); + for i in 0..65u32 { + words.extend([0x20000000, (i + 1) * 8]); + } + words.push(0x40000000); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Err(EINVAL)); + let mut words = Vec::new(); + for i in 0..64u32 { + words.extend([0x20000000, (i + 1) * 8]); + } + words.push(0x40000000); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); + } + /// The USC window of these cases and its first program, both in the mock's mapped range. + const USC: u64 = crate::command::SYNTHETIC; + /// A 24-byte indirect launch of the window's first program with its arguments at `args`. + fn indirect(args: u64) -> [u32; 6] { + [0x10080000, ((USC >> 38) as u32) << 22, (USC >> 6) as u32, + 0x40000000 | ((USC >> 40) & 0xff) as u32, (args >> 32) as u32 & 0xffff, args as u32] + } + /// `n` indirect launches, launch `i` with its arguments at `args(i)`, then TERMINATE. + fn launches(n: usize, args: impl Fn(usize) -> u64) -> Vec { + let mut words: Vec = (0..n).flat_map(|i| indirect(args(i))).collect(); + words.push(0x40000000); + words + } + fn covers(words: &[u32]) -> (Result, u32) { + crate::command::COVERS.with(|c| c.set(0)); + let r = gate::cdm(&mut vm(words), USC, 0); + (r, crate::command::COVERS.with(|c| c.get())) + } + const PAGE: u64 = mmu::UAT_PGSZ as u64; + #[test] + fn repeated_argument_pages_are_checked_once() { + // 1000 launches with their arguments in one page: the program page and that page. + let base = 2 * USC; + assert_eq!(covers(&launches(1000, |i| base + (i as u64 % 100) * 12)), (Ok(()), 2)); + // Alternating between two pages: one check per change of page. + assert_eq!(covers(&launches(10, |i| base + (i as u64 % 2) * PAGE)), (Ok(()), 11)); + // Arguments that cross a page are checked each time. + assert_eq!(covers(&launches(3, |_| base + PAGE - 8)), (Ok(()), 4)); + } + #[test] + fn page_check_budget_bounds_the_walk() { + let base = 2 * USC; + // The program page and 16383 argument pages: 16384 checks, admitted. + assert_eq!(covers(&launches(16383, |i| base + i as u64 * PAGE)), (Ok(()), 16384)); + // One more page: refused at the 16385th check, which is not made. + assert_eq!(covers(&launches(16384, |i| base + i as u64 * PAGE)), (Err(EINVAL), 16384)); + } + #[test] + fn a_large_stream_of_reused_pages_is_admitted() { + // 170000 launches (4 MB, inside the byte budget) over 16 argument pages. + let base = 2 * USC; + let words = launches(170000, |i| base + (i as u64 / 10625) * PAGE + (i as u64 % 1000) * 12); + assert_eq!(covers(&words), (Ok(()), 17)); } #[test] fn malformed_and_unreadable_streams_are_rejected() { - assert_eq!(gate::cdm(&vm(&[0xffffffff]), 0, 0), Err(EINVAL)); - assert_eq!(gate::cdm(&vm(&[0x20000000, 2]), 0, 0), Err(EINVAL)); - assert_eq!(gate::cdm(&mmu::Vm(vec![]), 0, 0), Err(EFAULT)); + assert_eq!(gate::cdm(&mut vm(&[0xffffffff]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&[0x20000000, 2]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut mmu::Vm(vec![]), 0, 0), Err(EFAULT)); } } ''' @@ -213,13 +298,44 @@ def main(): str(Path(__file__).with_name("pmp-bridge.c")), "-o", str(output)], check=True) subprocess.run([str(output)], check=True) + owner = (ROOT / "drivers/soc/apple/pmp-export.c").read_text() + start = owner.index("int apple_pmp_set_device_power(") + end = owner.index("\n}", start) + 2 + vote = owner[start:end] + start = owner.index("int apple_pmp_t6030_gpu_ready(") + end = owner.index("\n}", start) + 2 + (temp / "pmp-vote-production.h").write_text(vote + "\n" + owner[start:end]) + output = temp / "pmp-vote" + subprocess.run([os.environ.get("HOSTCC", "cc"), "-std=c11", "-Wall", "-Wextra", + "-Werror", "-I", str(temp), + str(Path(__file__).with_name("pmp-vote.c")), "-o", str(output)], check=True) + subprocess.run([str(output)], check=True) + # Native wire layout is a private per-backend facade. Its independent host test # checks that canonical normal/G17 UAPI remains unchanged; there is no global # Rust bindgen header selector in this integration. harness = RUST_PREFIX + f'\n#[path = "{G15 / "gate.rs"}"] mod gate;\n' + encode = (G15 / "encode.rs").read_text() + start = encode.index("pub(crate) const fn stamp_seed(") + end = encode.index("\n}", start) + 2 + harness += encode[start:end] + ''' +#[test] +fn firmware_stamp_wrap() { + for seed in [0xc1000000u32, 0x7a000000, 0x3d000000] { + let wrap = ((1u64 << 32) - seed as u64) / 0x100; + assert_eq!(stamp_seed(seed, 0), seed); + assert_eq!(stamp_seed(seed, wrap - 1), 0xffffff00); + assert_eq!(stamp_seed(seed, wrap), 0); + assert_eq!(stamp_seed(seed, wrap + 1), 0x100); + assert_eq!(stamp_seed(seed, 1 << 24), seed); + assert_eq!(stamp_seed(seed, u64::MAX), seed.wrapping_sub(0x100)); + } +} +''' harness += f'#[path = "{G15 / "recovery/logic.rs"}"] mod recovery;\n' harness += f'#[path = "{G15 / "power/idle.rs"}"] mod idle;\n' harness += f'#[path = "{G15 / "power/pstate.rs"}"] mod pstate;\n' + harness += f'#[path = "{G15 / "retain.rs"}"] mod retain;\n' harness += ("mod firmware_identity {\n" + f'#[path = "{G15 / "identity/matching.rs"}"] pub mod matching;\n' + f'#[path = "{G15 / "identity/lookup.rs"}"] pub mod lookup;\n' + "}\n") diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c b/tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c index 9909ac31e3a41..48e705c60b77f 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c @@ -8,22 +8,58 @@ typedef uint32_t u32; typedef uint64_t u64; struct device_node { + const char *name; + struct device_node *child, *next; bool available, reusable, extra, bad_reg; + u64 start, size; + int refs; }; struct resource { u64 start, end; }; -static struct device_node node = { .available = true }; +static struct device_node nodes[5]; +static struct device_node rmem; +#define node nodes[0] static bool missing; static u64 start = 0x100000, size = 0x4000; -static int refs; +static inline struct device_node *of_node_get(struct device_node *np) +{ + if (np) + np->refs++; + return np; +} +static void of_node_put(struct device_node *np); static struct device_node *of_find_node_by_path(const char *path) { - assert(!strncmp(path, "/reserved-memory/uat-", 21)); + const char *prefix = "/reserved-memory/"; + if (missing) return NULL; - refs++; - return &node; + if (!strcmp(path, "/reserved-memory")) + return of_node_get(&rmem); + assert(!strncmp(path, prefix, strlen(prefix))); + for (struct device_node *np = rmem.child; np; np = np->next) + if (!strcmp(np->name, path + strlen(prefix))) + return of_node_get(np); + return NULL; +} +static inline struct device_node *of_get_next_child(struct device_node *parent, + struct device_node *prev) +{ + struct device_node *next = NULL; + + if (parent) + next = prev ? prev->next : parent->child; + of_node_put(prev); + return of_node_get(next); +} +#define for_each_child_of_node(parent, child) \ + for ((child) = of_get_next_child((parent), NULL); (child); \ + (child) = of_get_next_child((parent), (child))) +static inline bool of_node_name_eq(struct device_node *np, const char *name) +{ + return np && strlen(name) == strcspn(np->name, "@") && + !strncmp(np->name, name, strlen(name)); } static bool of_device_is_available(struct device_node *np) { @@ -44,8 +80,8 @@ static int of_address_to_resource(struct device_node *np, int index, struct resource *res) { assert(index == 0); - res->start = start; - res->end = start + size - 1; + res->start = np->start; + res->end = np->start + np->size - 1; return np->bad_reg ? -1 : 0; } static u64 resource_size(struct resource *res) @@ -55,14 +91,49 @@ static u64 resource_size(struct resource *res) static void of_node_put(struct device_node *np) { if (np) - refs--; + assert(np->refs-- > 0); } #include "uat-production.h" int main(void) { + static const char *const names[] = { + "uat-ttbs", "uat-pagetables", "uat-handoff", "uat-pagetables-l2" + }; + static const char *const unit_names[] = { + "uat-ttbs@10", "uat-pagetables@20", "uat-handoff@30", "uat-pagetables-l2@40" + }; + rmem.child = nodes; + for (u32 i = 0; i < 4; ++i) { + nodes[i].available = true; + nodes[i].start = start + i * size; + nodes[i].size = size; + nodes[i].name = names[i]; + nodes[i].next = i < 3 ? &nodes[i + 1] : NULL; + } + for (u32 i = 0; i < 4; ++i) + assert(asahi_g15_reserved_uat(i, nodes[i].start, size)); for (u32 i = 0; i < 4; ++i) - assert(asahi_g15_reserved_uat(i, start, size)); + nodes[i].name = unit_names[i]; + for (u32 i = 0; i < 4; ++i) + assert(asahi_g15_reserved_uat(i, nodes[i].start, size)); + nodes[3].next = &nodes[4]; + nodes[4].name = "uat-ttbs@99"; + nodes[4].available = true; + assert(!asahi_g15_reserved_uat(0, start, size)); + nodes[4].available = false; + assert(!asahi_g15_reserved_uat(0, start, size)); + nodes[3].next = NULL; + node.name = "uat-ttbs-other@10"; + assert(!asahi_g15_reserved_uat(0, start, size)); + node.name = unit_names[0]; + for (u32 i = 0; i < 4; ++i) { + const char *saved = nodes[i].name; + + nodes[i].name = "unrelated@10"; + assert(!asahi_g15_reserved_uat(i, nodes[i].start, size)); + nodes[i].name = saved; + } assert(!asahi_g15_reserved_uat(4, start, size)); assert(!asahi_g15_reserved_uat(0, start + 0x4000, size)); assert(!asahi_g15_reserved_uat(0, start, size + 0x4000)); @@ -80,6 +151,8 @@ int main(void) node.extra = false; node.bad_reg = true; assert(!asahi_g15_reserved_uat(0, start, size)); - assert(refs == 0); - puts("production UAT owner: exact ranges, unavailable/reusable/malformed/extra reservations: PASS"); + assert(rmem.refs == 0); + for (u32 i = 0; i < ARRAY_SIZE(nodes); ++i) + assert(nodes[i].refs == 0); + puts("production UAT owner: unit addresses, unique owners and exact ranges: PASS"); } diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py index 787992e9b4c24..124314ac83a5d 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py @@ -70,7 +70,7 @@ def body(source,marker): bindings=r''' #[derive(Clone,Copy)]struct Token(u32);impl Token{fn last_slot(&self)->u32{self.0}} struct Slot;impl Slot{fn changed(&self)->bool{true}fn slot(&self)->u32{3}fn token(&self)->Token{Token(3)}} -struct Allocator;impl Allocator{fn set_limit(&self,_l:Option){}fn get(&self,_t:Option)->Result{Ok(Slot)}} +struct Allocator;impl Allocator{fn set_limit(&self,_l:Option){}fn get(&self,_t:Option)->Result{Ok(Slot)}fn try_get(&self,_t:Option)->Result{Ok(Slot)}} struct VmBinding{active_users:usize,bind_token:Option,binding:Option,generation:u8,ttb:u64} #[derive(Clone)]struct Vm{id:u64,binding:Rc>} struct VmBind(Vm,u32); @@ -78,7 +78,8 @@ def body(source,marker): mod module_parameters{pub struct Param;impl Param{pub fn value(&self)->u32{0}}pub static robust_isolation:Param=Param;} ''' helpers='\n'.join(body(mm,m) for m in ['const fn binding_wire_generation(', 'const fn application_ttb1(']) -bind='impl Uat{'+body(mm,'pub(crate) fn bind(&self, vm: &Vm)')+'}\n' +# bind() is the waiting form of bind_inner(); G15's try_bind() is the other. +bind='impl Uat{'+body(mm,'pub(crate) fn bind(&self, vm: &Vm)')+'\n'+body(mm,'fn bind_inner(&self, vm: &Vm, wait: bool)')+'}\n' vm=r''' struct VmInner{id:u64,binding:Rc>,uat_inner:Rc,page_table:UatPageTable} impl VmInner{fn ttb(&self)->u64{self.page_table.ttb}}