drm/asahi: g15: renumber the TA/FS and compute stamps before they wrap

drivers/gpu/drm/asahi/asahi_core.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 #![recursion_limit = "2048"]
 
@@ -161,6 +161,22 @@
             default: 1024,
             description: "G15: PB blocks posted above which an idle engine reclaims the PB",
         },
+        g15_stamp_base: u32 {
+            default: 0,
+            description: "G15 (testing): render ordinal the TA/FS stamps start from; 0 = unchanged",
+        },
+        g15_stamp_reseed_margin: u32 {
+            default: 0,
+            description: "G15: renders before the TA stamp wrap within which TA/FS are renumbered while idle; 0 = off",
+        },
+        g15_compute_stamp_base: u32 {
+            default: 0,
+            description: "G15 (testing): compute ordinal the compute stamp starts from; 0 = unchanged",
+        },
+        g15_compute_reseed_margin: u32 {
+            default: 0,
+            description: "G15: computes before the compute stamp wrap within which it is renumbered while idle; 0 = off",
+        },
         g15_early_tiling: u32 {
             default: 0,
             description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)",

drivers/gpu/drm/asahi/g15/encode.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Work-image and microsequence encoders.
@@ -69,6 +69,18 @@
     seed.wrapping_add((n as u32).wrapping_mul(0x100))
 }
 
+/// Ordinals after which [`stamp_seed`] repeats: 2^32 / 0x100.
+pub(crate) const STAMP_PERIOD: u64 = 1 << 24;
+
+/// Effective render ordinal at which the TA stamp wraps (0xffffff00 -> 0): 0x860000 with the TA
+/// seed. A wrap that skipped zero stopped the firmware ("DAG: Non Sequential Stamp Updates");
+/// the renumbering keeps the stamps away from it. The FS stamp wraps later (0xc30000).
+pub(crate) const RENDER_WRAP: u64 = 0xffffff - (hw::engine::TA_SEED / 0x100) as u64 + 1;
+
+/// Compute ordinal at which the compute stamp wraps (0xffffff00 -> 0): 0x3f0000 (4128768
+/// computes) with the compute seed, where the firmware may stop as at the TA stamp wrap.
+pub(crate) const COMPUTE_WRAP: u64 = STAMP_PERIOD - (hw::engine::COMPUTE_SEED / 0x100) as u64;
+
 /// Compute stamp of ordinal `n`.
 pub(crate) const fn stamp(n: u64) -> u32 {
     stamp_seed(hw::engine::COMPUTE_SEED, n)
@@ -101,8 +113,9 @@
     pub(crate) scratch: [u64; 5],
 }
 
-/// Encodes compute command `c` as ordinal `n`. The engine copies the result into the slot
-/// after zeroing the slot's images.
+/// Encodes compute command `c` as ordinal `n` with stamp `t` (the engine's renumbering moves
+/// the stamp off [`stamp`]`(n)`). The engine copies the result into the slot after zeroing the
+/// slot's images.
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn encode_compute(
     r: &ComputeRefs,
@@ -112,12 +125,12 @@
     ts_va: &[u64; 2],
     ctx: u32,
     n: u64,
+    t: u32,
     fault_mask: u32,
     w: &mut [u8; compute::WORK_SIZE],
     q: &mut [u8; compute::SEQ_SIZE],
 ) {
     let (wv, qv, h, pre) = (r.work, r.qinfo, r.hwdata, r.preempt_gva);
-    let t = stamp(n);
     let att_count = att.len().min(hw::sizes::MAX_ATTACHMENTS);
     // The saved-state pointers into the preemption buffer and the two 0x1c words; a SoC
     // encoded "fresh" carries zero in all six, as a first command does.

drivers/gpu/drm/asahi/g15/engine.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! The prefix/horizon/flight engine.
@@ -364,6 +364,12 @@
     /// Host compute ordinal and draw ordinal of the last publication.
     pub(crate) ordinal: u64,
     pub(crate) draw: u64,
+    /// Added to the draw ordinal before computing the TA/FS stamps; moved by
+    /// `renumber_render_stamps()`.
+    pub(crate) stamp_base: u64,
+    /// Added to the compute ordinal before computing the compute stamp; moved by
+    /// `renumber_compute_stamps()`.
+    pub(crate) compute_stamp_base: u64,
     /// Newest published receipts, the tail.
     pub(crate) target: u32,
     pub(crate) head: u32,
@@ -411,6 +417,8 @@
             shared: None,
             ordinal: 0,
             draw: 0,
+            stamp_base: params::stamp_base(),
+            compute_stamp_base: params::compute_stamp_base(),
             target: 0,
             head: 0,
             ta_target: 0,
@@ -560,12 +568,17 @@
                 break;
             };
             let job = self.engine.pending[index].clone();
+            // Inside a renumbering margin nothing is published behind a flight: the flights
+            // drain and the next publication renumbers the stamps while idle (once every ~8.7M
+            // renders or ~4.0M computes with margins of 100000).
+            let renumber_due = self.renumber_due();
             if let Some(tail) = self.engine.flights.last() {
                 if self.engine.flights.len() >= max_inflight
                     || !Arc::ptr_eq(&job.vm, &tail.job.vm)
                     || self.recovery.reinit
                     || self.recovery.fs_resync
                     || self.engine.rebuild_live
+                    || renumber_due
                 {
                     self.engine.stats.hold[0] += (self.engine.flights.len() >= max_inflight) as u64;
                     break;
@@ -708,13 +721,105 @@
         result
     }
 
+    /// The TA stamp is inside its renumbering margin.
+    fn render_renumber_due(&self) -> bool {
+        let margin = params::stamp_reseed_margin();
+        let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD;
+        margin != 0 && ordinal + margin >= encode::RENDER_WRAP
+    }
+
+    /// The compute stamp is inside its renumbering margin.
+    fn compute_renumber_due(&self) -> bool {
+        let margin = params::compute_reseed_margin();
+        let ordinal = (self.engine.ordinal + self.engine.compute_stamp_base) % encode::STAMP_PERIOD;
+        margin != 0 && ordinal + margin >= encode::COMPUTE_WRAP
+    }
+
+    /// Either stamp is inside its renumbering margin.
+    fn renumber_due(&self) -> bool {
+        self.render_renumber_due() || self.compute_renumber_due()
+    }
+
+    /// Computes and renders the current numbering has left before its compute and TA stamps
+    /// wrap (0 once wrapped), for `asahi-g15/stats`.
+    pub(crate) fn stamps_left(&self) -> (u64, u64) {
+        let period = encode::STAMP_PERIOD;
+        let c = (self.engine.ordinal + self.engine.compute_stamp_base) % period;
+        let r = (self.engine.draw + self.engine.stamp_base) % period;
+        (
+            encode::COMPUTE_WRAP.saturating_sub(c),
+            encode::RENDER_WRAP.saturating_sub(r),
+        )
+    }
+
+    /// Renumbers the TA/FS stamps from their seeds before the TA stamp wraps, after 0x860000
+    /// renders (about six hours of desktop), where the firmware may stop. Only called with
+    /// nothing in flight: the last flight retired with every stamp equal to its target, so both
+    /// queues are idle and the next render waits on exactly the seeds written here.
+    fn renumber_render_stamps(&mut self) -> Result {
+        if !self.render_renumber_due() {
+            return Ok(());
+        }
+        let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD;
+        let r = &self.graph.render;
+        for (e, seed) in [ids::TA_SEED, ids::FS_SEED].into_iter().enumerate() {
+            r.user_stamp[e].w32(0, seed)?;
+            r.fw_stamp[e].w32(0, seed)?;
+        }
+        // The next render (draw + 1) counts as ordinal 1 again: modulo the period of
+        // `stamp_seed`, whose ordinal 0 is the seeds just written.
+        let period = encode::STAMP_PERIOD;
+        self.engine.stamp_base = (period - self.engine.draw % period) % period;
+        self.engine.ta_target = ids::TA_SEED;
+        self.engine.fs_target = ids::FS_SEED;
+        self.engine.tail_fs = ids::FS_SEED;
+        dev_info!(
+            self.graph.dev.as_ref(),
+            "G15: TA/FS stamps renumbered from the seeds at draw {} (ordinal {:#x})\n",
+            self.engine.draw,
+            ordinal
+        );
+        Ok(())
+    }
+
+    /// Renumbers the compute stamp from its seed before it wraps, after 0x3f0000 computes,
+    /// where the firmware may stop. Like the TA/FS renumbering only with nothing in flight: the
+    /// compute queue is idle with both stamps at its target. The stamps are written back as
+    /// `ComputeGraph::init()` leaves them with no stamp base, so the next compute is ordinal 1
+    /// on the stamps of a fresh boot.
+    fn renumber_compute_stamps(&mut self) -> Result {
+        if !self.compute_renumber_due() {
+            return Ok(());
+        }
+        let ordinal = (self.engine.ordinal + self.engine.compute_stamp_base) % encode::STAMP_PERIOD;
+        let g = &self.graph.compute;
+        g.user_stamp.w32(0, ids::COMPUTE_SEED)?;
+        g.fw_stamp.w32(0, 0)?;
+        // The next compute (ordinal + 1) counts as ordinal 1 again.
+        let period = encode::STAMP_PERIOD;
+        self.engine.compute_stamp_base = (period - self.engine.ordinal % period) % period;
+        self.engine.target = 0;
+        self.engine.tail_ct = 0;
+        dev_info!(
+            self.graph.dev.as_ref(),
+            "G15: compute stamp renumbered from the seed at compute {} (ordinal {:#x})\n",
+            self.engine.ordinal,
+            ordinal
+        );
+        Ok(())
+    }
+
     fn publish_prefix_into(&mut self, job: &Arc<Job>, f: &mut Flight) -> Result {
         // The PB backing is mapped into the job's VM before anything of the prefix is changed.
         // Nothing is published when that fails, so the error must not be EIO (which reports a
         // queue WRITE that advanced).
         self.map_pb(&job.vm)
             .map_err(|e| if e == EIO { EFAULT } else { e })?;
         let overlap = !self.engine.flights.is_empty();
+        if !overlap {
+            self.renumber_render_stamps()?;
+            self.renumber_compute_stamps()?;
+        }
         let mut mixed = tuning::MIXED_PREFIX;
         let mut pf = Prefix {
             n: self.engine.ordinal,
@@ -986,7 +1091,7 @@
             self.engine.head = pf.c_head;
         }
         if renders != 0 {
-            f.ta_target = encode::stamp_seed(ids::TA_SEED, pf.draw);
+            f.ta_target = encode::stamp_seed(ids::TA_SEED, pf.draw + self.engine.stamp_base);
             self.engine.ta_target = f.ta_target;
             f.fs_target = pf.last_fs;
             self.engine.fs_target = pf.last_fs;
@@ -1122,7 +1227,7 @@
         later_render: bool,
     ) -> Result {
         let n = pf.n + 1;
-        let t = encode::stamp(n);
+        let t = encode::stamp(n + self.engine.compute_stamp_base);
         let ctx = job.slot();
         let ts = &cmd.timestamps;
         let mut lap = Lap::start();
@@ -1166,6 +1271,7 @@
             ts,
             ctx,
             n,
+            t,
             tuning::COMPUTE_FAULT_MASK,
             image(w)?,
             image(q)?,
@@ -1265,8 +1371,9 @@
         let n = pf.draw + 1;
         let usc = job.queue.usc_base;
         let ctx = job.slot();
-        let tta = encode::stamp_seed(ids::TA_SEED, n);
-        let tfs = encode::stamp_seed(ids::FS_SEED, n);
+        let sn = n + self.engine.stamp_base;
+        let tta = encode::stamp_seed(ids::TA_SEED, sn);
+        let tfs = encode::stamp_seed(ids::FS_SEED, sn);
         // Early tiling: a later render member after a render of the same prefix whose render
         // dependencies are fragment-only (FRAGMENT_BARRIERS) and that has no start timestamps
         // lets its TA wait for the previous TA; its FS still waits for the previous FS, then
@@ -1365,7 +1472,7 @@
         } else if early {
             let b = encode::barrier(
                 r.fw_stamp[0].va,
-                encode::stamp_seed(ids::TA_SEED, n - 1),
+                encode::stamp_seed(ids::TA_SEED, sn - 1),
                 0,
                 tta,
                 RENDER_WAIT_UUID,
@@ -1385,10 +1492,9 @@
         } else {
             let mut prev_fs = if pf.prev != Prev::None {
                 pf.last_fs
-            } else if n == 1 {
-                ids::FS_SEED
             } else {
-                encode::stamp_seed(ids::FS_SEED, n - 1)
+                // With no offset, ordinal 0 is the FS seed itself (the first render).
+                encode::stamp_seed(ids::FS_SEED, sn - 1)
             };
             // Restricted type 0: predecessor render in this prefix, no InitBM/resync.
             let internal = pf.prev == Prev::Render

drivers/gpu/drm/asahi/g15/graph.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! The persistent compute and render graphs.
@@ -251,7 +251,8 @@
         q.w64(qinfo::UNK_AC, 4)?;
         self.qstate.w32(qstate::CAP, sizes::QUEUE_CAP)?;
 
-        self.user_stamp.w32(0, engine::COMPUTE_SEED)?;
+        let start = super::encode::stamp(super::params::compute_stamp_base());
+        self.user_stamp.w32(0, start)?;
         self.fw_stamp.w32(0, 0)
     }
 }
@@ -352,8 +353,9 @@
             q.w64(qinfo::UNK_AC, 4)?;
             self.qstate[e].w32(qstate::CAP, sizes::QUEUE_CAP)?;
             self.ring[e].fill(0, 0xcc, sizes::QUEUE_CAP as usize * 8)?;
-            self.user_stamp[e].w32(0, seed[e])?;
-            self.fw_stamp[e].w32(0, seed[e])?;
+            let start = super::encode::stamp_seed(seed[e], super::params::stamp_base());
+            self.user_stamp[e].w32(0, start)?;
+            self.fw_stamp[e].w32(0, start)?;
         }
         init_sched_ctx(&self.sched_ctx)?;
         init_notifier(&self.notifier, &self.threshold)?;

drivers/gpu/drm/asahi/g15/params.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Module parameters and the GET_PARAMS identity.
@@ -18,6 +18,46 @@
 /// updates); this bounds the kernel's staging allocation.
 pub(crate) const MAX_BINDS: u32 = 1 << 20;
 
+/// Render ordinal the TA/FS stamps start from. Only for testing the stamp wrap without hours of
+/// drawing; 0 keeps the stamps exactly as before.
+pub(crate) fn stamp_base() -> u64 {
+    (*module_parameters::g15_stamp_base.value() as u64) % super::encode::STAMP_PERIOD
+}
+
+/// Renders before the TA stamp wrap within which the TA/FS stamps are renumbered from their seeds,
+/// at the first moment with nothing in flight (0: never). At least one prefix of renders, so
+/// that the last prefix published outside the margin cannot cross the wrap; at most half the
+/// stamp range.
+pub(crate) fn stamp_reseed_margin() -> u64 {
+    match *module_parameters::g15_stamp_reseed_margin.value() as u64 {
+        0 => 0,
+        m => m.clamp(
+            hw::tuning::PREFIX_RENDERS as u64,
+            super::encode::RENDER_WRAP / 2,
+        ),
+    }
+}
+
+/// Compute ordinal the compute stamp starts from. Only for testing the stamp wrap without
+/// millions of computes; 0 keeps the stamp exactly as before.
+pub(crate) fn compute_stamp_base() -> u64 {
+    (*module_parameters::g15_compute_stamp_base.value() as u64) % super::encode::STAMP_PERIOD
+}
+
+/// Computes before the compute stamp wrap within which the compute stamp is renumbered from its
+/// seed, at the first moment with nothing in flight (0: never). At least one prefix of computes,
+/// so that the last prefix published outside the margin cannot cross the wrap; at most half the
+/// stamp range.
+pub(crate) fn compute_reseed_margin() -> u64 {
+    match *module_parameters::g15_compute_reseed_margin.value() as u64 {
+        0 => 0,
+        m => m.clamp(
+            hw::sizes::COMPUTE_SLOTS as u64,
+            super::encode::COMPUTE_WRAP / 2,
+        ),
+    }
+}
+
 /// Host deadline of a published prefix without progress, ms (at least 100).
 pub(crate) fn deadline_ms() -> u32 {
     (*module_parameters::g15_deadline_ms.value()).max(100)

drivers/gpu/drm/asahi/g15/poll.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! The poll work: drain, retire, growth, hang checks, publication; and the repoll timers.
@@ -724,6 +724,16 @@
             s.hz_ahead
         )?;
         self.pools.write_stats(f)?;
+        // What is left before the stamps wrap, and the renumbering margins (0: off).
+        let (compute_left, render_left) = self.stamps_left();
+        writeln!(
+            f,
+            "stamps: compute_left {} compute_margin {} render_left {} render_margin {}",
+            compute_left,
+            params::compute_reseed_margin(),
+            render_left,
+            params::stamp_reseed_margin()
+        )?;
         writeln!(
             f,
             "stage: ta_us {} fs_us {} cdm_us {} span_us {}",