diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 84a9bff28d6d75..8797d75bca1a6d 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -161,6 +161,22 @@ module! { default: 1024, description: "G15: PB blocks posted above which an idle engine reclaims the PB", }, + g15_stamp_base: u32 { + default: 0, + description: "G15 (testing): render ordinal the TA/FS stamps start from; 0 = unchanged", + }, + g15_stamp_reseed_margin: u32 { + default: 0, + description: "G15: renders before the TA stamp wrap within which TA/FS are renumbered while idle; 0 = off", + }, + g15_compute_stamp_base: u32 { + default: 0, + description: "G15 (testing): compute ordinal the compute stamp starts from; 0 = unchanged", + }, + g15_compute_reseed_margin: u32 { + default: 0, + description: "G15: computes before the compute stamp wrap within which it is renumbered while idle; 0 = off", + }, g15_early_tiling: u32 { default: 0, description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)", diff --git a/drivers/gpu/drm/asahi/g15/encode.rs b/drivers/gpu/drm/asahi/g15/encode.rs index 64aebf32548914..a62cca8613b332 100644 --- a/drivers/gpu/drm/asahi/g15/encode.rs +++ b/drivers/gpu/drm/asahi/g15/encode.rs @@ -69,6 +69,18 @@ pub(crate) const fn stamp_seed(seed: u32, n: u64) -> u32 { seed.wrapping_add((n as u32).wrapping_mul(0x100)) } +/// Ordinals after which [`stamp_seed`] repeats: 2^32 / 0x100. +pub(crate) const STAMP_PERIOD: u64 = 1 << 24; + +/// Effective render ordinal at which the TA stamp wraps (0xffffff00 -> 0): 0x860000 with the TA +/// seed. A wrap that skipped zero stopped the firmware ("DAG: Non Sequential Stamp Updates"); +/// the renumbering keeps the stamps away from it. The FS stamp wraps later (0xc30000). +pub(crate) const RENDER_WRAP: u64 = 0xffffff - (hw::engine::TA_SEED / 0x100) as u64 + 1; + +/// Compute ordinal at which the compute stamp wraps (0xffffff00 -> 0): 0x3f0000 (4128768 +/// computes) with the compute seed, where the firmware may stop as at the TA stamp wrap. +pub(crate) const COMPUTE_WRAP: u64 = STAMP_PERIOD - (hw::engine::COMPUTE_SEED / 0x100) as u64; + /// Compute stamp of ordinal `n`. pub(crate) const fn stamp(n: u64) -> u32 { stamp_seed(hw::engine::COMPUTE_SEED, n) @@ -101,8 +113,9 @@ pub(crate) struct ComputeRefs { pub(crate) scratch: [u64; 5], } -/// Encodes compute command `c` as ordinal `n`. The engine copies the result into the slot -/// after zeroing the slot's images. +/// Encodes compute command `c` as ordinal `n` with stamp `t` (the engine's renumbering moves +/// the stamp off [`stamp`]`(n)`). The engine copies the result into the slot after zeroing the +/// slot's images. #[allow(clippy::too_many_arguments)] pub(crate) fn encode_compute( r: &ComputeRefs, @@ -112,12 +125,12 @@ pub(crate) fn encode_compute( ts_va: &[u64; 2], ctx: u32, n: u64, + t: u32, fault_mask: u32, w: &mut [u8; compute::WORK_SIZE], q: &mut [u8; compute::SEQ_SIZE], ) { let (wv, qv, h, pre) = (r.work, r.qinfo, r.hwdata, r.preempt_gva); - let t = stamp(n); let att_count = att.len().min(hw::sizes::MAX_ATTACHMENTS); // The saved-state pointers into the preemption buffer and the two 0x1c words; a SoC // encoded "fresh" carries zero in all six, as a first command does. diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 80e4a31a78706d..3b0636ae7743e5 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -364,6 +364,12 @@ pub(crate) struct Engine { /// Host compute ordinal and draw ordinal of the last publication. pub(crate) ordinal: u64, pub(crate) draw: u64, + /// Added to the draw ordinal before computing the TA/FS stamps; moved by + /// `renumber_render_stamps()`. + pub(crate) stamp_base: u64, + /// Added to the compute ordinal before computing the compute stamp; moved by + /// `renumber_compute_stamps()`. + pub(crate) compute_stamp_base: u64, /// Newest published receipts, the tail. pub(crate) target: u32, pub(crate) head: u32, @@ -411,6 +417,8 @@ impl Engine { shared: None, ordinal: 0, draw: 0, + stamp_base: params::stamp_base(), + compute_stamp_base: params::compute_stamp_base(), target: 0, head: 0, ta_target: 0, @@ -560,12 +568,17 @@ impl Firmware { break; }; let job = self.engine.pending[index].clone(); + // Inside a renumbering margin nothing is published behind a flight: the flights + // drain and the next publication renumbers the stamps while idle (once every ~8.7M + // renders or ~4.0M computes with margins of 100000). + let renumber_due = self.renumber_due(); if let Some(tail) = self.engine.flights.last() { if self.engine.flights.len() >= max_inflight || !Arc::ptr_eq(&job.vm, &tail.job.vm) || self.recovery.reinit || self.recovery.fs_resync || self.engine.rebuild_live + || renumber_due { self.engine.stats.hold[0] += (self.engine.flights.len() >= max_inflight) as u64; break; @@ -708,6 +721,94 @@ impl Firmware { result } + /// The TA stamp is inside its renumbering margin. + fn render_renumber_due(&self) -> bool { + let margin = params::stamp_reseed_margin(); + let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD; + margin != 0 && ordinal + margin >= encode::RENDER_WRAP + } + + /// The compute stamp is inside its renumbering margin. + fn compute_renumber_due(&self) -> bool { + let margin = params::compute_reseed_margin(); + let ordinal = (self.engine.ordinal + self.engine.compute_stamp_base) % encode::STAMP_PERIOD; + margin != 0 && ordinal + margin >= encode::COMPUTE_WRAP + } + + /// Either stamp is inside its renumbering margin. + fn renumber_due(&self) -> bool { + self.render_renumber_due() || self.compute_renumber_due() + } + + /// Computes and renders the current numbering has left before its compute and TA stamps + /// wrap (0 once wrapped), for `asahi-g15/stats`. + pub(crate) fn stamps_left(&self) -> (u64, u64) { + let period = encode::STAMP_PERIOD; + let c = (self.engine.ordinal + self.engine.compute_stamp_base) % period; + let r = (self.engine.draw + self.engine.stamp_base) % period; + ( + encode::COMPUTE_WRAP.saturating_sub(c), + encode::RENDER_WRAP.saturating_sub(r), + ) + } + + /// Renumbers the TA/FS stamps from their seeds before the TA stamp wraps, after 0x860000 + /// renders (about six hours of desktop), where the firmware may stop. Only called with + /// nothing in flight: the last flight retired with every stamp equal to its target, so both + /// queues are idle and the next render waits on exactly the seeds written here. + fn renumber_render_stamps(&mut self) -> Result { + if !self.render_renumber_due() { + return Ok(()); + } + let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD; + let r = &self.graph.render; + for (e, seed) in [ids::TA_SEED, ids::FS_SEED].into_iter().enumerate() { + r.user_stamp[e].w32(0, seed)?; + r.fw_stamp[e].w32(0, seed)?; + } + // The next render (draw + 1) counts as ordinal 1 again: modulo the period of + // `stamp_seed`, whose ordinal 0 is the seeds just written. + let period = encode::STAMP_PERIOD; + self.engine.stamp_base = (period - self.engine.draw % period) % period; + self.engine.ta_target = ids::TA_SEED; + self.engine.fs_target = ids::FS_SEED; + self.engine.tail_fs = ids::FS_SEED; + dev_info!( + self.graph.dev.as_ref(), + "G15: TA/FS stamps renumbered from the seeds at draw {} (ordinal {:#x})\n", + self.engine.draw, + ordinal + ); + Ok(()) + } + + /// Renumbers the compute stamp from its seed before it wraps, after 0x3f0000 computes, + /// where the firmware may stop. Like the TA/FS renumbering only with nothing in flight: the + /// compute queue is idle with both stamps at its target. The stamps are written back as + /// `ComputeGraph::init()` leaves them with no stamp base, so the next compute is ordinal 1 + /// on the stamps of a fresh boot. + fn renumber_compute_stamps(&mut self) -> Result { + if !self.compute_renumber_due() { + return Ok(()); + } + let ordinal = (self.engine.ordinal + self.engine.compute_stamp_base) % encode::STAMP_PERIOD; + let g = &self.graph.compute; + g.user_stamp.w32(0, ids::COMPUTE_SEED)?; + g.fw_stamp.w32(0, 0)?; + // The next compute (ordinal + 1) counts as ordinal 1 again. + let period = encode::STAMP_PERIOD; + self.engine.compute_stamp_base = (period - self.engine.ordinal % period) % period; + self.engine.target = 0; + self.engine.tail_ct = 0; + dev_info!( + self.graph.dev.as_ref(), + "G15: compute stamp renumbered from the seed at compute {} (ordinal {:#x})\n", + self.engine.ordinal, + ordinal + ); + Ok(()) + } + fn publish_prefix_into(&mut self, job: &Arc, f: &mut Flight) -> Result { // The PB backing is mapped into the job's VM before anything of the prefix is changed. // Nothing is published when that fails, so the error must not be EIO (which reports a @@ -715,6 +816,10 @@ impl Firmware { self.map_pb(&job.vm) .map_err(|e| if e == EIO { EFAULT } else { e })?; let overlap = !self.engine.flights.is_empty(); + if !overlap { + self.renumber_render_stamps()?; + self.renumber_compute_stamps()?; + } let mut mixed = tuning::MIXED_PREFIX; let mut pf = Prefix { n: self.engine.ordinal, @@ -986,7 +1091,7 @@ impl Firmware { self.engine.head = pf.c_head; } if renders != 0 { - f.ta_target = encode::stamp_seed(ids::TA_SEED, pf.draw); + f.ta_target = encode::stamp_seed(ids::TA_SEED, pf.draw + self.engine.stamp_base); self.engine.ta_target = f.ta_target; f.fs_target = pf.last_fs; self.engine.fs_target = pf.last_fs; @@ -1122,7 +1227,7 @@ impl Firmware { later_render: bool, ) -> Result { let n = pf.n + 1; - let t = encode::stamp(n); + let t = encode::stamp(n + self.engine.compute_stamp_base); let ctx = job.slot(); let ts = &cmd.timestamps; let mut lap = Lap::start(); @@ -1166,6 +1271,7 @@ impl Firmware { ts, ctx, n, + t, tuning::COMPUTE_FAULT_MASK, image(w)?, image(q)?, @@ -1265,8 +1371,9 @@ impl Firmware { let n = pf.draw + 1; let usc = job.queue.usc_base; let ctx = job.slot(); - let tta = encode::stamp_seed(ids::TA_SEED, n); - let tfs = encode::stamp_seed(ids::FS_SEED, n); + let sn = n + self.engine.stamp_base; + let tta = encode::stamp_seed(ids::TA_SEED, sn); + let tfs = encode::stamp_seed(ids::FS_SEED, sn); // Early tiling: a later render member after a render of the same prefix whose render // dependencies are fragment-only (FRAGMENT_BARRIERS) and that has no start timestamps // lets its TA wait for the previous TA; its FS still waits for the previous FS, then @@ -1365,7 +1472,7 @@ impl Firmware { } else if early { let b = encode::barrier( r.fw_stamp[0].va, - encode::stamp_seed(ids::TA_SEED, n - 1), + encode::stamp_seed(ids::TA_SEED, sn - 1), 0, tta, RENDER_WAIT_UUID, @@ -1385,10 +1492,9 @@ impl Firmware { } else { let mut prev_fs = if pf.prev != Prev::None { pf.last_fs - } else if n == 1 { - ids::FS_SEED } else { - encode::stamp_seed(ids::FS_SEED, n - 1) + // With no offset, ordinal 0 is the FS seed itself (the first render). + encode::stamp_seed(ids::FS_SEED, sn - 1) }; // Restricted type 0: predecessor render in this prefix, no InitBM/resync. let internal = pf.prev == Prev::Render diff --git a/drivers/gpu/drm/asahi/g15/graph.rs b/drivers/gpu/drm/asahi/g15/graph.rs index 9a62f25f19a63b..18680fd05596f8 100644 --- a/drivers/gpu/drm/asahi/g15/graph.rs +++ b/drivers/gpu/drm/asahi/g15/graph.rs @@ -251,7 +251,8 @@ impl ComputeGraph { q.w64(qinfo::UNK_AC, 4)?; self.qstate.w32(qstate::CAP, sizes::QUEUE_CAP)?; - self.user_stamp.w32(0, engine::COMPUTE_SEED)?; + let start = super::encode::stamp(super::params::compute_stamp_base()); + self.user_stamp.w32(0, start)?; self.fw_stamp.w32(0, 0) } } @@ -352,8 +353,9 @@ impl RenderGraph { q.w64(qinfo::UNK_AC, 4)?; self.qstate[e].w32(qstate::CAP, sizes::QUEUE_CAP)?; self.ring[e].fill(0, 0xcc, sizes::QUEUE_CAP as usize * 8)?; - self.user_stamp[e].w32(0, seed[e])?; - self.fw_stamp[e].w32(0, seed[e])?; + let start = super::encode::stamp_seed(seed[e], super::params::stamp_base()); + self.user_stamp[e].w32(0, start)?; + self.fw_stamp[e].w32(0, start)?; } init_sched_ctx(&self.sched_ctx)?; init_notifier(&self.notifier, &self.threshold)?; diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index 48d21e45d0acd6..858cec0a291c5e 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -18,6 +18,46 @@ use crate::module_parameters; /// updates); this bounds the kernel's staging allocation. pub(crate) const MAX_BINDS: u32 = 1 << 20; +/// Render ordinal the TA/FS stamps start from. Only for testing the stamp wrap without hours of +/// drawing; 0 keeps the stamps exactly as before. +pub(crate) fn stamp_base() -> u64 { + (*module_parameters::g15_stamp_base.value() as u64) % super::encode::STAMP_PERIOD +} + +/// Renders before the TA stamp wrap within which the TA/FS stamps are renumbered from their seeds, +/// at the first moment with nothing in flight (0: never). At least one prefix of renders, so +/// that the last prefix published outside the margin cannot cross the wrap; at most half the +/// stamp range. +pub(crate) fn stamp_reseed_margin() -> u64 { + match *module_parameters::g15_stamp_reseed_margin.value() as u64 { + 0 => 0, + m => m.clamp( + hw::tuning::PREFIX_RENDERS as u64, + super::encode::RENDER_WRAP / 2, + ), + } +} + +/// Compute ordinal the compute stamp starts from. Only for testing the stamp wrap without +/// millions of computes; 0 keeps the stamp exactly as before. +pub(crate) fn compute_stamp_base() -> u64 { + (*module_parameters::g15_compute_stamp_base.value() as u64) % super::encode::STAMP_PERIOD +} + +/// Computes before the compute stamp wrap within which the compute stamp is renumbered from its +/// seed, at the first moment with nothing in flight (0: never). At least one prefix of computes, +/// so that the last prefix published outside the margin cannot cross the wrap; at most half the +/// stamp range. +pub(crate) fn compute_reseed_margin() -> u64 { + match *module_parameters::g15_compute_reseed_margin.value() as u64 { + 0 => 0, + m => m.clamp( + hw::sizes::COMPUTE_SLOTS as u64, + super::encode::COMPUTE_WRAP / 2, + ), + } +} + /// Host deadline of a published prefix without progress, ms (at least 100). pub(crate) fn deadline_ms() -> u32 { (*module_parameters::g15_deadline_ms.value()).max(100) diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 647c503ba55037..158abab719b52e 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -724,6 +724,16 @@ impl Firmware { s.hz_ahead )?; self.pools.write_stats(f)?; + // What is left before the stamps wrap, and the renumbering margins (0: off). + let (compute_left, render_left) = self.stamps_left(); + writeln!( + f, + "stamps: compute_left {} compute_margin {} render_left {} render_margin {}", + compute_left, + params::compute_reseed_margin(), + render_left, + params::stamp_reseed_margin() + )?; writeln!( f, "stage: ta_us {} fs_us {} cdm_us {} span_us {}",