From 41c359da81755d96315191bc650872e4ec92d94a Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Fri, 13 Nov 2026 10:49:20 -0500 Subject: [PATCH 1/2] soc: apple: mailbox: receive T6050 GPU mailboxes by interrupt once it fires The T6050 GPU mailboxes receive by polling, because the recv-not-empty interrupt does not reach the AIC on the 14-inch M5 Max MacBook Pro: a 25 us hrtimer for 1 ms after each send, otherwise a once-per-jiffy poller, and the interrupt is never enabled. On the 16-inch M5 Max MacBook Pro the interrupt works, and there polling costs about 5% of GPU throughput in an inference engine's decode step. Enable the recv-not-empty interrupt on these mailboxes again, and keep both pollers running beside it until it has delivered a message. From then on the fast poller stops and sends no longer arm it: the interrupt receives, as on the other ASC mailboxes. The line is level triggered and the GPU handler is oneshot, so a message that arrives while it runs fires it again. Each mailbox start proves the line again. The jiffy poller keeps running as a watchdog. Where the interrupt delivers, the poller finds the FIFO empty. A message it does find was not delivered by the interrupt (or its thread had not run yet): the poller hands it on and clears the proof, so that the sends after it arm the fast poller again until the interrupt delivers a message again. A line that never fires keeps polling exactly as before. A line that delivers once and then stops, or fires only spuriously, loses no message: the watchdog takes one that it misses on a following pass, normally the next, subject to workqueue scheduling and to its budget of 64 messages per pass. The ASCWrap v8 storm guard now covers these mailboxes too: a line that fires 1000 times in a row with nothing to read is disabled at the AIC and receive falls back to polling. The hardware flag rx_poll_only becomes rx_poll_until_irq. A full send FIFO is still waited out by sleeping polls, without the send-empty line, as before. Tested with KUnit (arm64, QEMU, suite apple_mbox_g17_stall): new cases cover proof by a delivered message, the watchdog taking the proof back when the line is silent with a message queued, the storm guard, and start and stop. They wait on what the pollers receive, and read state after cancelling them or under rx_lock. On an M5 Max MacBook Pro (J716c, whose interrupt fires), an inference engine's decode, two runs each without and with this commit: GPU time of its verification pass 21.93 / 21.93 -> 21.02 / 21.04 ms, single-token decode 62.4 / 63.8 -> 66.4 / 67.2 tok/s, output identical; the GPU mailboxes' recv interrupts count in /proc/interrupts, the watchdog took the proof back 64 times in ~100 s of decode (each time the next message proved the line again), and the storm guard did not trip. Developed with Claude Code. Signed-off-by: Erik Bourget --- drivers/soc/apple/mailbox-g17-test.h | 349 +++++++++++++++++++++++++++ drivers/soc/apple/mailbox.c | 141 ++++++++--- 2 files changed, 456 insertions(+), 34 deletions(-) diff --git a/drivers/soc/apple/mailbox-g17-test.h b/drivers/soc/apple/mailbox-g17-test.h index 10690d1d27bf7..dbefe13f2eedf 100644 --- a/drivers/soc/apple/mailbox-g17-test.h +++ b/drivers/soc/apple/mailbox-g17-test.h @@ -3,6 +3,8 @@ * The receive FIFO never empties and the send FIFO never gains space. */ #include +#include +#include static void apple_g17_test_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie) @@ -93,10 +95,357 @@ static void apple_g17_rx_util_min_test(struct kunit *test) #endif } +/* T6050 receive until the recv interrupt is proven. The interrupt is + * requested as the driver requests it, on a descriptor with no controller + * behind it: the tests enable and disable it and call the receive handler as + * its thread would. The registers are RAM; + * apple_g17_rx_fifo() makes the receive FIFO never empty or always empty. + * Whether a poller runs is shown by what it receives: the test thread itself + * receives only through the calls it makes, and snapshots state after + * cancelling the pollers or under rx_lock. + */ +static const struct apple_mbox_hw apple_g17_rx_until_irq_hw = { + .i2a_control = 0, .i2a_recv0 = 8, .i2a_recv1 = 16, + .a2i_control = 32, .a2i_send0 = 40, .a2i_send1 = 48, + .control_empty = BIT(17), .control_full = BIT(16), + .rx_poll_until_irq = true, +}; + +/* Generous: a poller that runs receives within a jiffy or a fast-poll period. */ +#define APPLE_G17_TEST_RX_WAIT msecs_to_jiffies(5000) + +struct apple_g17_rx_fixture { + struct apple_mbox_priv *priv; + /* rx_lock: messages received, by any path. */ + unsigned int count; + /* Completed by every message received. */ + struct completion rx; +}; + +static void apple_g17_fixture_rx(struct apple_mbox *mbox, + struct apple_mbox_msg msg, void *cookie) +{ + struct apple_g17_rx_fixture *f = cookie; + + f->count++; + complete(&f->rx); +} + +static void apple_g17_rx_fixture_exit(void *data) +{ + struct apple_mbox_priv *priv = data; + + cancel_delayed_work_sync(&priv->rx_poll_work); + hrtimer_cancel(&priv->rx_fast_timer); + free_irq(priv->mbox.irq_recv_not_empty, &priv->mbox); + irq_free_desc(priv->mbox.irq_recv_not_empty); +} + +/* The fixture outlives the test's stack, like the pollers. */ +static struct apple_g17_rx_fixture *apple_g17_rx_fixture(struct kunit *test) +{ + struct apple_g17_rx_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct apple_mbox_priv *priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + void *regs = kunit_kzalloc(test, 64, GFP_KERNEL); + struct apple_mbox *mbox; + int irq; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, priv); + KUNIT_ASSERT_NOT_NULL(test, regs); + irq = irq_alloc_desc(NUMA_NO_NODE); + KUNIT_ASSERT_GE(test, irq, 0); + irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); + init_completion(&f->rx); + mbox = &priv->mbox; + mbox->hw = &apple_g17_rx_until_irq_hw; + mbox->regs = (void __iomem *)regs; + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; + mbox->rx = apple_g17_fixture_rx; + mbox->cookie = f; + mbox->irq_recv_not_empty = irq; + mbox->irq_send_empty = -1; + mbox->active = true; + spin_lock_init(&mbox->rx_lock); + spin_lock_init(&mbox->tx_lock); + hrtimer_setup(&priv->rx_fast_timer, apple_mbox_fast_rx_poll, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + INIT_DELAYED_WORK(&priv->rx_poll_work, apple_mbox_v8_rx_poll_work); + if (request_threaded_irq(irq, NULL, apple_mbox_recv_irq, + IRQF_ONESHOT | IRQF_NO_AUTOEN, "apple-mbox-test", + mbox)) { + irq_free_desc(irq); + KUNIT_FAIL_AND_ABORT(test, "request_threaded_irq failed"); + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + apple_g17_rx_fixture_exit, priv), 0); + f->priv = priv; + return f; +} + +/* Under rx_lock, so a running poller sees the FIFO before or after the change. */ +static void apple_g17_rx_fifo(struct apple_mbox *mbox, bool full) +{ + unsigned long flags; + + spin_lock_irqsave(&mbox->rx_lock, flags); + writel(full ? 0 : mbox->hw->control_empty, mbox->regs + mbox->hw->i2a_control); + spin_unlock_irqrestore(&mbox->rx_lock, flags); +} + +/* + * Fill the FIFO and forget earlier receives, both under rx_lock: whatever + * completes f->rx afterwards received from this full FIFO. + */ +static void apple_g17_rx_fill(struct apple_g17_rx_fixture *f) +{ + struct apple_mbox *mbox = &f->priv->mbox; + unsigned long flags; + + spin_lock_irqsave(&mbox->rx_lock, flags); + reinit_completion(&f->rx); + writel(0, mbox->regs + mbox->hw->i2a_control); + spin_unlock_irqrestore(&mbox->rx_lock, flags); +} + +static bool apple_g17_rx_received(struct apple_g17_rx_fixture *f) +{ + return wait_for_completion_timeout(&f->rx, APPLE_G17_TEST_RX_WAIT); +} + +static unsigned int apple_g17_rx_count(struct apple_g17_rx_fixture *f) +{ + struct apple_mbox *mbox = &f->priv->mbox; + unsigned long flags; + unsigned int count; + + spin_lock_irqsave(&mbox->rx_lock, flags); + count = f->count; + spin_unlock_irqrestore(&mbox->rx_lock, flags); + return count; +} + +static bool apple_g17_rx_irq_enabled(struct apple_mbox *mbox) +{ + return !irqd_irq_disabled(irq_get_irq_data(mbox->irq_recv_not_empty)); +} + +/* + * A send arms the fast poller: the armed poller fires and receives from the + * full FIFO. Call with the jiffy poller cancelled, so that nothing else + * receives. + */ +static bool apple_g17_send_fast_polls(struct kunit *test, + struct apple_g17_rx_fixture *f) +{ + struct apple_mbox_msg msg = { 0 }; + bool received; + + apple_g17_rx_fill(f); + KUNIT_EXPECT_EQ(test, apple_mbox_send(&f->priv->mbox, msg, true), 0); + received = apple_g17_rx_received(f); + hrtimer_cancel(&f->priv->rx_fast_timer); + apple_g17_rx_fifo(&f->priv->mbox, false); + return received; +} + +/* A send that does not arm the fast poller sets no deadline and starts no timer. */ +static bool apple_g17_send_leaves_fast_poller(struct kunit *test, + struct apple_mbox_priv *priv) +{ + struct apple_mbox_msg msg = { 0 }; + bool left; + + WRITE_ONCE(priv->rx_fast_until, 0); + KUNIT_EXPECT_EQ(test, apple_mbox_send(&priv->mbox, msg, true), 0); + left = !READ_ONCE(priv->rx_fast_until) && + !hrtimer_active(&priv->rx_fast_timer); + hrtimer_cancel(&priv->rx_fast_timer); + return left; +} + +/* Proof: both pollers run until the live interrupt delivers a message. */ +static void apple_g17_rx_irq_proof_test(struct kunit *test) +{ + struct apple_g17_rx_fixture *f = apple_g17_rx_fixture(test); + struct apple_mbox_priv *priv = f->priv; + struct apple_mbox *mbox = &priv->mbox; + unsigned int before; + + priv->rx_irq_live = true; + + /* Not proven: the fast poller restarts and a send arms it. Receiving + * moves the deadline, so the restart does not depend on the clock. + */ + apple_g17_rx_fifo(mbox, true); + KUNIT_EXPECT_EQ(test, apple_mbox_fast_rx_poll(&priv->rx_fast_timer), + HRTIMER_RESTART); + KUNIT_EXPECT_EQ(test, f->count, APPLE_G17_MBOX_RX_BUDGET); + KUNIT_EXPECT_TRUE(test, apple_g17_send_fast_polls(test, f)); + + /* An interrupt that finds nothing proves nothing; one with a message does. */ + apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); + KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); + KUNIT_EXPECT_EQ(test, priv->rx_irq_empty_run, 1U); + before = f->count; + apple_g17_rx_fifo(mbox, true); + apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); + KUNIT_EXPECT_TRUE(test, priv->rx_irq_proven); + KUNIT_EXPECT_EQ(test, priv->rx_irq_empty_run, 0U); + KUNIT_EXPECT_EQ(test, f->count, before + APPLE_G17_MBOX_RX_BUDGET); + + /* Proven: the fast poller stops and a send does not arm it. */ + KUNIT_EXPECT_EQ(test, apple_mbox_fast_rx_poll(&priv->rx_fast_timer), + HRTIMER_NORESTART); + apple_g17_rx_fifo(mbox, false); + KUNIT_EXPECT_TRUE(test, apple_g17_send_leaves_fast_poller(test, priv)); + + /* A line that is not live (stopped, or cut by the guard) proves nothing. */ + priv->rx_irq_proven = false; + priv->rx_irq_live = false; + apple_g17_rx_fifo(mbox, true); + apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); + KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); +} + +/* After proof the jiffy poller keeps running. A message it finds that the + * interrupt did not deliver takes the proof back; the interrupt proves the + * line again. + */ +static void apple_g17_rx_watchdog_test(struct kunit *test) +{ + struct apple_g17_rx_fixture *f = apple_g17_rx_fixture(test); + struct apple_mbox_priv *priv = f->priv; + struct apple_mbox *mbox = &priv->mbox; + unsigned long flags; + unsigned int before; + bool proven; + + priv->rx_irq_live = true; + apple_g17_rx_fifo(mbox, true); + apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); + KUNIT_ASSERT_TRUE(test, priv->rx_irq_proven); + + /* The interrupt drained the FIFO: the watchdog finds nothing. */ + apple_g17_rx_fifo(mbox, false); + apple_mbox_v8_rx_poll_work(&priv->rx_poll_work.work); + spin_lock_irqsave(&mbox->rx_lock, flags); + proven = priv->rx_irq_proven; + spin_unlock_irqrestore(&mbox->rx_lock, flags); + KUNIT_EXPECT_TRUE(test, proven); + + /* It queued itself again. The line goes silent with a message queued: + * a later pass takes the message and the proof. + */ + before = apple_g17_rx_count(f); + apple_g17_rx_fill(f); + KUNIT_EXPECT_TRUE(test, apple_g17_rx_received(f)); + cancel_delayed_work_sync(&priv->rx_poll_work); + KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); + KUNIT_EXPECT_GE(test, f->count, before + APPLE_G17_MBOX_RX_BUDGET); + + /* Back to polling: the fast poller restarts and a send arms it. */ + apple_g17_rx_fifo(mbox, false); + KUNIT_EXPECT_TRUE(test, apple_g17_send_fast_polls(test, f)); + apple_g17_rx_fifo(mbox, true); + KUNIT_EXPECT_EQ(test, apple_mbox_fast_rx_poll(&priv->rx_fast_timer), + HRTIMER_RESTART); + + /* The interrupt delivers again and proves the line again. */ + apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); + KUNIT_EXPECT_TRUE(test, priv->rx_irq_proven); + apple_g17_rx_fifo(mbox, false); + KUNIT_EXPECT_TRUE(test, apple_g17_send_leaves_fast_poller(test, priv)); +} + +/* The storm guard cuts a proven line that fires with nothing to read, starts + * the jiffy poller and takes the proof back, so a send fast-polls again. + */ +static void apple_g17_rx_storm_guard_test(struct kunit *test) +{ + struct apple_g17_rx_fixture *f = apple_g17_rx_fixture(test); + struct apple_mbox_priv *priv = f->priv; + struct apple_mbox *mbox = &priv->mbox; + unsigned int n; + + enable_irq(mbox->irq_recv_not_empty); + priv->rx_irq_live = true; + apple_g17_rx_fifo(mbox, true); + apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); + KUNIT_ASSERT_TRUE(test, priv->rx_irq_proven); + + apple_g17_rx_fifo(mbox, false); + for (n = 1; n < APPLE_ASCWRAP_V8_RX_STORM_LIMIT; n++) + apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); + KUNIT_EXPECT_TRUE(test, priv->rx_irq_live); + KUNIT_EXPECT_TRUE(test, priv->rx_irq_proven); + KUNIT_EXPECT_TRUE(test, apple_g17_rx_irq_enabled(mbox)); + /* Nothing queued the jiffy poller yet. */ + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&priv->rx_poll_work)); + + apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); + KUNIT_EXPECT_FALSE(test, priv->rx_irq_live); + KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); + KUNIT_EXPECT_FALSE(test, apple_g17_rx_irq_enabled(mbox)); + + /* The guard queued the jiffy poller: it takes a message left queued. */ + apple_g17_rx_fill(f); + KUNIT_EXPECT_TRUE(test, apple_g17_rx_received(f)); + cancel_delayed_work_sync(&priv->rx_poll_work); + apple_g17_rx_fifo(mbox, false); + KUNIT_EXPECT_TRUE(test, apple_g17_send_fast_polls(test, f)); + + /* The cut line proves nothing. */ + apple_g17_rx_fifo(mbox, true); + apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); + KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); +} + +/* Start enables the line, starts the jiffy poller and clears the proof; + * stop disables the line, stops the poller and clears the proof. + */ +static void apple_g17_rx_start_stop_test(struct kunit *test) +{ + struct apple_g17_rx_fixture *f = apple_g17_rx_fixture(test); + struct apple_mbox_priv *priv = f->priv; + struct apple_mbox *mbox = &priv->mbox; + unsigned long flags; + bool proven; + + apple_g17_rx_fifo(mbox, false); + priv->rx_irq_proven = true; + apple_mbox_v8_rx_start(mbox); + KUNIT_EXPECT_TRUE(test, priv->rx_irq_live); + KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); + KUNIT_EXPECT_TRUE(test, apple_g17_rx_irq_enabled(mbox)); + + /* Start queued the jiffy poller: it takes a message left queued. */ + apple_g17_rx_fill(f); + KUNIT_EXPECT_TRUE(test, apple_g17_rx_received(f)); + apple_g17_rx_fifo(mbox, false); + + spin_lock_irqsave(&mbox->rx_lock, flags); + apple_mbox_rx_irq_guard(mbox, 1); + proven = priv->rx_irq_proven; + spin_unlock_irqrestore(&mbox->rx_lock, flags); + KUNIT_EXPECT_TRUE(test, proven); + + mbox->active = false; + apple_mbox_v8_rx_stop(mbox); + KUNIT_EXPECT_FALSE(test, priv->rx_irq_live); + KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); + KUNIT_EXPECT_FALSE(test, apple_g17_rx_irq_enabled(mbox)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&priv->rx_poll_work)); +} + static struct kunit_case apple_g17_stall_cases[] = { KUNIT_CASE(apple_g17_rx_flood_bounded_test), KUNIT_CASE(apple_g17_tx_stall_bounded_test), KUNIT_CASE(apple_g17_rx_util_min_test), + KUNIT_CASE(apple_g17_rx_irq_proof_test), + KUNIT_CASE(apple_g17_rx_watchdog_test), + KUNIT_CASE(apple_g17_rx_storm_guard_test), + KUNIT_CASE(apple_g17_rx_start_stop_test), { } }; static struct kunit_suite apple_g17_stall_suite = { diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index b73e7da0b2b59..879bcb7c826a4 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -212,11 +212,12 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == #define APPLE_G17_MBOX_RX_BUDGET 64 #define APPLE_G17_MBOX_ATOMIC_TX_US 1000 /* - * T6050 poll-only receive: every send arms an hrtimer that polls the receive - * FIFO at this period, kept for this long after the last message or send. A - * GPU reply lands within one 0.22 ms frame; polling every 25 us bounds the - * added receive latency to ~11% of a frame instead of one 4 ms jiffy, while - * idle receive stays on the jiffy poller. + * T6050 receive until the recv-not-empty interrupt has delivered a message: + * every send arms an hrtimer that polls the receive FIFO at this period, kept + * for this long after the last message or send. A GPU reply lands within one + * 0.22 ms frame; polling every 25 us bounds the added receive latency to ~11% + * of a frame instead of one 4 ms jiffy, while idle receive stays on the jiffy + * poller. */ #define APPLE_MBOX_RX_FAST_POLL_US 25 #define APPLE_MBOX_RX_FAST_GRACE_US 1000 @@ -236,8 +237,12 @@ struct apple_mbox_hw { unsigned int num_ascwrap_v6_roles; /* T8152 GPU ASCWrap v8: wrapper + iop-vbar + post-office window + PSD. */ bool is_ascwrap_v8; - /* The mailbox interrupt never reaches the AIC: receive by polling. */ - bool rx_poll_only; + /* + * The recv-not-empty interrupt does not reach the AIC on every board: + * enable it, and also receive by polling until it delivers a message. + * The jiffy poller keeps running after that as a watchdog. + */ + bool rx_poll_until_irq; /* Status word counts RX occupancy and free TX slots; paired transfers. */ bool counted_fifo; bool ap_initializes_mailboxes; @@ -275,6 +280,13 @@ struct apple_mbox_priv { bool rx_poll_only; /* rx_lock: recv interrupt enabled and not yet disabled again. */ bool rx_irq_live; + /* + * rx_lock, read lock-free by the send path: the live recv interrupt + * delivered a message, so fast receive polling stops. Only set while + * rx_irq_live, and cleared with it, or when the jiffy poller finds a + * message the interrupt did not deliver. + */ + bool rx_irq_proven; /* rx_lock: consecutive recv interrupts that drained nothing. */ unsigned int rx_irq_empty_run; /* rx_lock: an unpowered-PSD receive skip was logged this activation. */ @@ -293,7 +305,7 @@ static struct apple_mbox_priv *apple_mbox_to_priv(struct apple_mbox *mbox) /* Only providers allocated by apple_mbox_probe() have a receive poller. */ static bool apple_mbox_has_rx_poller(const struct apple_mbox *mbox) { - return mbox->hw->is_ascwrap_v8 || mbox->hw->rx_poll_only; + return mbox->hw->is_ascwrap_v8 || mbox->hw->rx_poll_until_irq; } /* Both ASCWrap generations share the consumer lifecycle API below. */ @@ -666,11 +678,10 @@ static struct apple_mbox_msg apple_mbox_read_msg(struct apple_mbox *mbox) static int apple_mbox_poll_locked(struct apple_mbox *mbox); /* - * Fast receive tick for rx_poll_only mailboxes (T6050): the same receive + * Fast receive tick for rx_poll_until_irq mailboxes (T6050): the same receive * step apple_mbox_v8_rx_poll_work() runs, under the same rx_lock, with - * apple_mbox_poll_locked() doing the same PSD and removal checks. The v8 - * storm guard does not apply: it guards the recv-not-empty IRQ line, which - * rx_poll_only hardware never enables. Runs in hardirq context. + * apple_mbox_poll_locked() doing the same removal checks. It stops while the + * recv interrupt is proven. Runs in hardirq context. */ static enum hrtimer_restart apple_mbox_fast_rx_poll(struct hrtimer *timer) { @@ -680,6 +691,7 @@ static enum hrtimer_restart apple_mbox_fast_rx_poll(struct hrtimer *timer) unsigned long flags; ktime_t now; int consumed = 0; + bool proven; if (!READ_ONCE(mbox->active) || READ_ONCE(mbox->removing)) return HRTIMER_NORESTART; @@ -687,8 +699,13 @@ static enum hrtimer_restart apple_mbox_fast_rx_poll(struct hrtimer *timer) spin_lock_irqsave(&mbox->rx_lock, flags); if (likely(!READ_ONCE(mbox->removing))) consumed = apple_mbox_poll_locked(mbox); + proven = priv->rx_irq_proven; spin_unlock_irqrestore(&mbox->rx_lock, flags); + /* The interrupt receives (see apple_mbox_rx_irq_guard()). */ + if (proven) + return HRTIMER_NORESTART; + now = ktime_get(); if (consumed > 0) { /* Traffic arrived: keep polling as if a send just happened. */ @@ -708,10 +725,13 @@ static enum hrtimer_restart apple_mbox_fast_rx_poll(struct hrtimer *timer) * tx_lock under rx_lock. Kick never touches rx_lock, so no ordering exists * to invert. A kick that loses a race with the timer's final no-restart * decision leaves the fast poller disarmed once; the next jiffy poll or - * send recovers it. + * send recovers it. A stale rx_irq_proven costs at most one fast-poll + * burst, or one reply left to the jiffy poller. */ static void apple_mbox_fast_rx_kick(struct apple_mbox_priv *priv) { + if (READ_ONCE(priv->rx_irq_proven)) + return; WRITE_ONCE(priv->rx_fast_until, ktime_add_us(ktime_get(), APPLE_MBOX_RX_FAST_GRACE_US)); /* Publish the deadline before deciding to arm. */ @@ -755,9 +775,10 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, } remaining = deadline - now; if (!atomic && apple_mbox_has_rx_poller(mbox) && - apple_mbox_to_priv(mbox)->rx_poll_only) { + (mbox->hw->rx_poll_until_irq || + apple_mbox_to_priv(mbox)->rx_poll_only)) { /* - * Poll-only v8: never enable the send-empty line. + * Poll-only v8, or T6050: never enable the send-empty line. * Sleep-poll the TX status, touching MMIO only under * tx_lock so that removal cannot unmap it underneath. */ @@ -846,7 +867,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, apple_mbox_write_msg(mbox, msg); /* The reply may be back well within a jiffy: fast-poll for it. */ - if (mbox->hw->rx_poll_only) + if (mbox->hw->rx_poll_until_irq) apple_mbox_fast_rx_kick(apple_mbox_to_priv(mbox)); spin_unlock_irqrestore(&mbox->tx_lock, flags); @@ -968,17 +989,32 @@ int apple_mbox_set_rx_util_min(struct apple_mbox *mbox, int util_min) EXPORT_SYMBOL_GPL(apple_mbox_set_rx_util_min); /* - * ASCWrap v8 storm guard, rx_lock held. The handler cannot mask the line at - * the mailbox (no irq controls), so a line that keeps firing with nothing to - * read (for example a send-empty line wired as recv-not-empty) would lock up - * the CPU while always returning IRQ_HANDLED. Cut it at the AIC and poll. + * Storm guard of the mailboxes with a receive poller, rx_lock held. The + * handler cannot mask the line at the mailbox (no irq controls), so a line + * that keeps firing with nothing to read (for example a send-empty line + * wired as recv-not-empty) would lock up the CPU while always returning + * IRQ_HANDLED. Cut it at the AIC and poll. + * + * A live line that delivers a message has proven itself: fast polling stops + * and the line receives, with the jiffy poller left as a watchdog (see + * apple_mbox_rx_watchdog()). It is level triggered and the G17 handler is + * oneshot, so a message that arrives while the handler runs fires it again. + * If the line later storms, the guard below cuts it and makes sure the jiffy + * poller runs; the next send restarts fast polling. */ -static void apple_mbox_v8_rx_irq_guard(struct apple_mbox *mbox, int drained) +static void apple_mbox_rx_irq_guard(struct apple_mbox *mbox, int drained) { struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); if (drained > 0) { priv->rx_irq_empty_run = 0; + if (priv->rx_irq_live && !priv->rx_irq_proven) { + WRITE_ONCE(priv->rx_irq_proven, true); + if (mbox->hw->rx_poll_until_irq) + dev_dbg(mbox->dev, + "recv-not-empty IRQ %d delivered a message; fast receive polling stopped\n", + mbox->irq_recv_not_empty); + } return; } if (!priv->rx_irq_live || @@ -986,9 +1022,10 @@ static void apple_mbox_v8_rx_irq_guard(struct apple_mbox *mbox, int drained) return; priv->rx_irq_live = false; + WRITE_ONCE(priv->rx_irq_proven, false); disable_irq_nosync(mbox->irq_recv_not_empty); dev_err(mbox->dev, - "ASCWrap v8 recv-not-empty IRQ %d fired %u times without a message; disabled, receive falls back to polling\n", + "recv-not-empty IRQ %d fired %u times without a message; disabled, receive falls back to polling\n", mbox->irq_recv_not_empty, priv->rx_irq_empty_run); queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, 0); } @@ -1006,8 +1043,8 @@ static irqreturn_t apple_mbox_recv_irq(int irq, void *data) spin_lock_irqsave(&mbox->rx_lock, flags); if (likely(!READ_ONCE(mbox->removing))) consumed = apple_mbox_poll_locked(mbox); - if (mbox->hw->is_ascwrap_v8) - apple_mbox_v8_rx_irq_guard(mbox, consumed); + if (apple_mbox_has_rx_poller(mbox)) + apple_mbox_rx_irq_guard(mbox, consumed); spin_unlock_irqrestore(&mbox->rx_lock, flags); /* G17 uses a oneshot threaded IRQ. Leave it masked during this sleep @@ -1085,17 +1122,51 @@ int apple_mbox_poll_one(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_poll_one); -/* ASCWrap v8 receive poller: poll-only mode, or after the storm guard. */ +/* + * Watchdog after each jiffy poll, rx_lock held. On a line that delivers, the + * interrupt thread has drained the FIFO and the poller finds it empty. A + * message the poller consumed while the interrupt was proven was not + * delivered by the interrupt, or its thread had not run yet: either way + * receive goes back to fast polling on every send until the interrupt + * delivers a message again. All receive steps drain under rx_lock, so + * messages reach the consumer once and in FIFO order whichever step reads + * them. + */ +static void apple_mbox_rx_watchdog(struct apple_mbox *mbox, int consumed) +{ + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + + if (consumed <= 0 || !priv->rx_irq_proven) + return; + WRITE_ONCE(priv->rx_irq_proven, false); + dev_dbg(mbox->dev, + "recv-not-empty IRQ %d left a message in the FIFO; receive polling restarted\n", + mbox->irq_recv_not_empty); +} + +/* + * Receive poller: v8 poll-only mode, T6050 for as long as the mailbox is + * active (as a watchdog once the recv interrupt is proven), or after the + * storm guard. + */ static void apple_mbox_v8_rx_poll_work(struct work_struct *work) { struct apple_mbox_priv *priv = container_of(to_delayed_work(work), struct apple_mbox_priv, rx_poll_work); struct apple_mbox *mbox = &priv->mbox; + unsigned long flags; + int consumed = 0; if (!READ_ONCE(mbox->active) || READ_ONCE(mbox->removing)) return; - apple_mbox_poll(mbox); + + spin_lock_irqsave(&mbox->rx_lock, flags); + if (likely(!READ_ONCE(mbox->removing))) + consumed = apple_mbox_poll_locked(mbox); + apple_mbox_rx_watchdog(mbox, consumed); + spin_unlock_irqrestore(&mbox->rx_lock, flags); + queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, APPLE_ASCWRAP_V8_RX_POLL_JIFFIES); } @@ -1110,11 +1181,13 @@ static void apple_mbox_v8_rx_start(struct apple_mbox *mbox) priv->rx_irq_empty_run = 0; priv->rx_psd_skip_logged = false; priv->rx_irq_live = !priv->rx_poll_only; + /* Each activation proves the line again. */ + WRITE_ONCE(priv->rx_irq_proven, false); spin_unlock_irqrestore(&mbox->rx_lock, flags); - if (priv->rx_poll_only) + if (priv->rx_poll_only || mbox->hw->rx_poll_until_irq) queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, 0); - else + if (!priv->rx_poll_only) enable_irq(mbox->irq_recv_not_empty); } @@ -1133,12 +1206,13 @@ static void apple_mbox_v8_rx_stop(struct apple_mbox *mbox) spin_lock_irqsave(&mbox->rx_lock, flags); live = priv->rx_irq_live; priv->rx_irq_live = false; + WRITE_ONCE(priv->rx_irq_proven, false); spin_unlock_irqrestore(&mbox->rx_lock, flags); if (live) disable_irq(mbox->irq_recv_not_empty); /* Same cancel-and-drain guarantee for the fast poller as below. */ - if (mbox->hw->rx_poll_only) + if (mbox->hw->rx_poll_until_irq) hrtimer_cancel(&priv->rx_fast_timer); cancel_delayed_work_sync(&priv->rx_poll_work); } @@ -1327,7 +1401,7 @@ static void apple_mbox_quiesce(void *data) } /* The v8 poller also stops on the removal fence; make sure it is gone. */ if (apple_mbox_has_rx_poller(mbox)) { - if (mbox->hw->rx_poll_only) + if (mbox->hw->rx_poll_until_irq) hrtimer_cancel(&apple_mbox_to_priv(mbox)->rx_fast_timer); cancel_delayed_work_sync(&apple_mbox_to_priv(mbox)->rx_poll_work); } @@ -1546,8 +1620,7 @@ static int apple_mbox_probe(struct platform_device *pdev) return -EINVAL; if (mbox->hw->counted_fifo && !IS_ENABLED(CONFIG_ARM64)) return -EOPNOTSUPP; - if (mbox->hw->rx_poll_only) { - priv->rx_poll_only = true; + if (mbox->hw->rx_poll_until_irq) { hrtimer_setup(&priv->rx_fast_timer, apple_mbox_fast_rx_poll, CLOCK_MONOTONIC, HRTIMER_MODE_REL); INIT_DELAYED_WORK(&priv->rx_poll_work, apple_mbox_v8_rx_poll_work); @@ -1942,7 +2015,7 @@ static const struct apple_mbox_hw apple_mbox_t6050_ascwrap_v6_hw = { .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, .has_irq_controls = false, - .rx_poll_only = true, + .rx_poll_until_irq = true, }; static const struct apple_mbox_hw apple_mbox_t6050_j775d_ascwrap_v6_hw = { @@ -1959,7 +2032,7 @@ static const struct apple_mbox_hw apple_mbox_t6050_j775d_ascwrap_v6_hw = { .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, .has_irq_controls = false, - .rx_poll_only = true, + .rx_poll_until_irq = true, }; /* From 788441e0890e18493d72e7df1445f01a844f1061 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 13 Nov 2026 12:02:31 -0500 Subject: [PATCH 2/2] soc: apple: mailbox: receive T6050 GPU mailboxes by interrupt ASC mailboxes are interrupt driven; T6050's GPU mailboxes now use the same paths as the other ASCWrap v6 GPU mailboxes. The three T6050 GPU compatibles (apple,t6050-ascwrap-v6, apple,t6050-x1n1-ascwrap-v6 and apple,t6050-j775d-ascwrap-v6) describe the ASC v4 mailbox block behind the ASCWrap v6 wrapper offset, as the T8140, T8142 and T6040 GPU mailboxes do. Their hardware descriptions now differ from those only in the wrapper windows, and they take the same start, stop, receive and send paths: - the recv-not-empty line is the threaded, oneshot GPU receive interrupt (IRQF_ONESHOT | IRQF_NO_AUTOEN | IRQF_NO_SUSPEND), enabled by apple_mbox_start() and disabled by apple_mbox_stop(); - its handler drains the I2A FIFO until it is empty, one receive budget per pass; - a non-atomic sender that finds the A2I FIFO full waits for the send-empty interrupt and its completion. Remove what received them by polling instead: - the rx_poll_until_irq hardware flag (rx_poll_only before); - the 25 us receive hrtimer, its kick from apple_mbox_send() and its deadline; - the jiffy receive poller and its watchdog for these mailboxes, and the rx_irq_proven state; - the storm guard's extension to them; - the sleeping poll of a full send FIFO, which left the send-empty interrupt unused. This undoes, for these mailboxes, the receive and send changes of commits 43c6ca839e8b ("soc: apple: mailbox: receive T6050 GPU mailboxes by polling"), a23755c12ee9 ("soc: apple: mailbox: poll T6050 GPU receive quickly while a reply is outstanding") and 41c359da8175 ("soc: apple: mailbox: receive T6050 GPU mailboxes by interrupt once it fires"). The T8152 ASCWrap v8 provider keeps its receive poller, its poll-only default, the ascwrap_v8_rx_irq parameter and its storm guard, unchanged. Drop the KUnit cases that tested the removed receive polling. The apple_mbox_g17_stall suite keeps its receive budget, send stall and receive utilization cases. No exported interface changes. Signed-off-by: Diljit Singh --- drivers/soc/apple/mailbox-g17-test.h | 349 --------------------------- drivers/soc/apple/mailbox.c | 197 ++------------- 2 files changed, 16 insertions(+), 530 deletions(-) diff --git a/drivers/soc/apple/mailbox-g17-test.h b/drivers/soc/apple/mailbox-g17-test.h index dbefe13f2eedf..10690d1d27bf7 100644 --- a/drivers/soc/apple/mailbox-g17-test.h +++ b/drivers/soc/apple/mailbox-g17-test.h @@ -3,8 +3,6 @@ * The receive FIFO never empties and the send FIFO never gains space. */ #include -#include -#include static void apple_g17_test_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie) @@ -95,357 +93,10 @@ static void apple_g17_rx_util_min_test(struct kunit *test) #endif } -/* T6050 receive until the recv interrupt is proven. The interrupt is - * requested as the driver requests it, on a descriptor with no controller - * behind it: the tests enable and disable it and call the receive handler as - * its thread would. The registers are RAM; - * apple_g17_rx_fifo() makes the receive FIFO never empty or always empty. - * Whether a poller runs is shown by what it receives: the test thread itself - * receives only through the calls it makes, and snapshots state after - * cancelling the pollers or under rx_lock. - */ -static const struct apple_mbox_hw apple_g17_rx_until_irq_hw = { - .i2a_control = 0, .i2a_recv0 = 8, .i2a_recv1 = 16, - .a2i_control = 32, .a2i_send0 = 40, .a2i_send1 = 48, - .control_empty = BIT(17), .control_full = BIT(16), - .rx_poll_until_irq = true, -}; - -/* Generous: a poller that runs receives within a jiffy or a fast-poll period. */ -#define APPLE_G17_TEST_RX_WAIT msecs_to_jiffies(5000) - -struct apple_g17_rx_fixture { - struct apple_mbox_priv *priv; - /* rx_lock: messages received, by any path. */ - unsigned int count; - /* Completed by every message received. */ - struct completion rx; -}; - -static void apple_g17_fixture_rx(struct apple_mbox *mbox, - struct apple_mbox_msg msg, void *cookie) -{ - struct apple_g17_rx_fixture *f = cookie; - - f->count++; - complete(&f->rx); -} - -static void apple_g17_rx_fixture_exit(void *data) -{ - struct apple_mbox_priv *priv = data; - - cancel_delayed_work_sync(&priv->rx_poll_work); - hrtimer_cancel(&priv->rx_fast_timer); - free_irq(priv->mbox.irq_recv_not_empty, &priv->mbox); - irq_free_desc(priv->mbox.irq_recv_not_empty); -} - -/* The fixture outlives the test's stack, like the pollers. */ -static struct apple_g17_rx_fixture *apple_g17_rx_fixture(struct kunit *test) -{ - struct apple_g17_rx_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); - struct apple_mbox_priv *priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); - void *regs = kunit_kzalloc(test, 64, GFP_KERNEL); - struct apple_mbox *mbox; - int irq; - - KUNIT_ASSERT_NOT_NULL(test, f); - KUNIT_ASSERT_NOT_NULL(test, priv); - KUNIT_ASSERT_NOT_NULL(test, regs); - irq = irq_alloc_desc(NUMA_NO_NODE); - KUNIT_ASSERT_GE(test, irq, 0); - irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); - init_completion(&f->rx); - mbox = &priv->mbox; - mbox->hw = &apple_g17_rx_until_irq_hw; - mbox->regs = (void __iomem *)regs; - mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; - mbox->rx = apple_g17_fixture_rx; - mbox->cookie = f; - mbox->irq_recv_not_empty = irq; - mbox->irq_send_empty = -1; - mbox->active = true; - spin_lock_init(&mbox->rx_lock); - spin_lock_init(&mbox->tx_lock); - hrtimer_setup(&priv->rx_fast_timer, apple_mbox_fast_rx_poll, - CLOCK_MONOTONIC, HRTIMER_MODE_REL); - INIT_DELAYED_WORK(&priv->rx_poll_work, apple_mbox_v8_rx_poll_work); - if (request_threaded_irq(irq, NULL, apple_mbox_recv_irq, - IRQF_ONESHOT | IRQF_NO_AUTOEN, "apple-mbox-test", - mbox)) { - irq_free_desc(irq); - KUNIT_FAIL_AND_ABORT(test, "request_threaded_irq failed"); - } - KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, - apple_g17_rx_fixture_exit, priv), 0); - f->priv = priv; - return f; -} - -/* Under rx_lock, so a running poller sees the FIFO before or after the change. */ -static void apple_g17_rx_fifo(struct apple_mbox *mbox, bool full) -{ - unsigned long flags; - - spin_lock_irqsave(&mbox->rx_lock, flags); - writel(full ? 0 : mbox->hw->control_empty, mbox->regs + mbox->hw->i2a_control); - spin_unlock_irqrestore(&mbox->rx_lock, flags); -} - -/* - * Fill the FIFO and forget earlier receives, both under rx_lock: whatever - * completes f->rx afterwards received from this full FIFO. - */ -static void apple_g17_rx_fill(struct apple_g17_rx_fixture *f) -{ - struct apple_mbox *mbox = &f->priv->mbox; - unsigned long flags; - - spin_lock_irqsave(&mbox->rx_lock, flags); - reinit_completion(&f->rx); - writel(0, mbox->regs + mbox->hw->i2a_control); - spin_unlock_irqrestore(&mbox->rx_lock, flags); -} - -static bool apple_g17_rx_received(struct apple_g17_rx_fixture *f) -{ - return wait_for_completion_timeout(&f->rx, APPLE_G17_TEST_RX_WAIT); -} - -static unsigned int apple_g17_rx_count(struct apple_g17_rx_fixture *f) -{ - struct apple_mbox *mbox = &f->priv->mbox; - unsigned long flags; - unsigned int count; - - spin_lock_irqsave(&mbox->rx_lock, flags); - count = f->count; - spin_unlock_irqrestore(&mbox->rx_lock, flags); - return count; -} - -static bool apple_g17_rx_irq_enabled(struct apple_mbox *mbox) -{ - return !irqd_irq_disabled(irq_get_irq_data(mbox->irq_recv_not_empty)); -} - -/* - * A send arms the fast poller: the armed poller fires and receives from the - * full FIFO. Call with the jiffy poller cancelled, so that nothing else - * receives. - */ -static bool apple_g17_send_fast_polls(struct kunit *test, - struct apple_g17_rx_fixture *f) -{ - struct apple_mbox_msg msg = { 0 }; - bool received; - - apple_g17_rx_fill(f); - KUNIT_EXPECT_EQ(test, apple_mbox_send(&f->priv->mbox, msg, true), 0); - received = apple_g17_rx_received(f); - hrtimer_cancel(&f->priv->rx_fast_timer); - apple_g17_rx_fifo(&f->priv->mbox, false); - return received; -} - -/* A send that does not arm the fast poller sets no deadline and starts no timer. */ -static bool apple_g17_send_leaves_fast_poller(struct kunit *test, - struct apple_mbox_priv *priv) -{ - struct apple_mbox_msg msg = { 0 }; - bool left; - - WRITE_ONCE(priv->rx_fast_until, 0); - KUNIT_EXPECT_EQ(test, apple_mbox_send(&priv->mbox, msg, true), 0); - left = !READ_ONCE(priv->rx_fast_until) && - !hrtimer_active(&priv->rx_fast_timer); - hrtimer_cancel(&priv->rx_fast_timer); - return left; -} - -/* Proof: both pollers run until the live interrupt delivers a message. */ -static void apple_g17_rx_irq_proof_test(struct kunit *test) -{ - struct apple_g17_rx_fixture *f = apple_g17_rx_fixture(test); - struct apple_mbox_priv *priv = f->priv; - struct apple_mbox *mbox = &priv->mbox; - unsigned int before; - - priv->rx_irq_live = true; - - /* Not proven: the fast poller restarts and a send arms it. Receiving - * moves the deadline, so the restart does not depend on the clock. - */ - apple_g17_rx_fifo(mbox, true); - KUNIT_EXPECT_EQ(test, apple_mbox_fast_rx_poll(&priv->rx_fast_timer), - HRTIMER_RESTART); - KUNIT_EXPECT_EQ(test, f->count, APPLE_G17_MBOX_RX_BUDGET); - KUNIT_EXPECT_TRUE(test, apple_g17_send_fast_polls(test, f)); - - /* An interrupt that finds nothing proves nothing; one with a message does. */ - apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); - KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); - KUNIT_EXPECT_EQ(test, priv->rx_irq_empty_run, 1U); - before = f->count; - apple_g17_rx_fifo(mbox, true); - apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); - KUNIT_EXPECT_TRUE(test, priv->rx_irq_proven); - KUNIT_EXPECT_EQ(test, priv->rx_irq_empty_run, 0U); - KUNIT_EXPECT_EQ(test, f->count, before + APPLE_G17_MBOX_RX_BUDGET); - - /* Proven: the fast poller stops and a send does not arm it. */ - KUNIT_EXPECT_EQ(test, apple_mbox_fast_rx_poll(&priv->rx_fast_timer), - HRTIMER_NORESTART); - apple_g17_rx_fifo(mbox, false); - KUNIT_EXPECT_TRUE(test, apple_g17_send_leaves_fast_poller(test, priv)); - - /* A line that is not live (stopped, or cut by the guard) proves nothing. */ - priv->rx_irq_proven = false; - priv->rx_irq_live = false; - apple_g17_rx_fifo(mbox, true); - apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); - KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); -} - -/* After proof the jiffy poller keeps running. A message it finds that the - * interrupt did not deliver takes the proof back; the interrupt proves the - * line again. - */ -static void apple_g17_rx_watchdog_test(struct kunit *test) -{ - struct apple_g17_rx_fixture *f = apple_g17_rx_fixture(test); - struct apple_mbox_priv *priv = f->priv; - struct apple_mbox *mbox = &priv->mbox; - unsigned long flags; - unsigned int before; - bool proven; - - priv->rx_irq_live = true; - apple_g17_rx_fifo(mbox, true); - apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); - KUNIT_ASSERT_TRUE(test, priv->rx_irq_proven); - - /* The interrupt drained the FIFO: the watchdog finds nothing. */ - apple_g17_rx_fifo(mbox, false); - apple_mbox_v8_rx_poll_work(&priv->rx_poll_work.work); - spin_lock_irqsave(&mbox->rx_lock, flags); - proven = priv->rx_irq_proven; - spin_unlock_irqrestore(&mbox->rx_lock, flags); - KUNIT_EXPECT_TRUE(test, proven); - - /* It queued itself again. The line goes silent with a message queued: - * a later pass takes the message and the proof. - */ - before = apple_g17_rx_count(f); - apple_g17_rx_fill(f); - KUNIT_EXPECT_TRUE(test, apple_g17_rx_received(f)); - cancel_delayed_work_sync(&priv->rx_poll_work); - KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); - KUNIT_EXPECT_GE(test, f->count, before + APPLE_G17_MBOX_RX_BUDGET); - - /* Back to polling: the fast poller restarts and a send arms it. */ - apple_g17_rx_fifo(mbox, false); - KUNIT_EXPECT_TRUE(test, apple_g17_send_fast_polls(test, f)); - apple_g17_rx_fifo(mbox, true); - KUNIT_EXPECT_EQ(test, apple_mbox_fast_rx_poll(&priv->rx_fast_timer), - HRTIMER_RESTART); - - /* The interrupt delivers again and proves the line again. */ - apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); - KUNIT_EXPECT_TRUE(test, priv->rx_irq_proven); - apple_g17_rx_fifo(mbox, false); - KUNIT_EXPECT_TRUE(test, apple_g17_send_leaves_fast_poller(test, priv)); -} - -/* The storm guard cuts a proven line that fires with nothing to read, starts - * the jiffy poller and takes the proof back, so a send fast-polls again. - */ -static void apple_g17_rx_storm_guard_test(struct kunit *test) -{ - struct apple_g17_rx_fixture *f = apple_g17_rx_fixture(test); - struct apple_mbox_priv *priv = f->priv; - struct apple_mbox *mbox = &priv->mbox; - unsigned int n; - - enable_irq(mbox->irq_recv_not_empty); - priv->rx_irq_live = true; - apple_g17_rx_fifo(mbox, true); - apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); - KUNIT_ASSERT_TRUE(test, priv->rx_irq_proven); - - apple_g17_rx_fifo(mbox, false); - for (n = 1; n < APPLE_ASCWRAP_V8_RX_STORM_LIMIT; n++) - apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); - KUNIT_EXPECT_TRUE(test, priv->rx_irq_live); - KUNIT_EXPECT_TRUE(test, priv->rx_irq_proven); - KUNIT_EXPECT_TRUE(test, apple_g17_rx_irq_enabled(mbox)); - /* Nothing queued the jiffy poller yet. */ - KUNIT_EXPECT_FALSE(test, delayed_work_pending(&priv->rx_poll_work)); - - apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); - KUNIT_EXPECT_FALSE(test, priv->rx_irq_live); - KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); - KUNIT_EXPECT_FALSE(test, apple_g17_rx_irq_enabled(mbox)); - - /* The guard queued the jiffy poller: it takes a message left queued. */ - apple_g17_rx_fill(f); - KUNIT_EXPECT_TRUE(test, apple_g17_rx_received(f)); - cancel_delayed_work_sync(&priv->rx_poll_work); - apple_g17_rx_fifo(mbox, false); - KUNIT_EXPECT_TRUE(test, apple_g17_send_fast_polls(test, f)); - - /* The cut line proves nothing. */ - apple_g17_rx_fifo(mbox, true); - apple_mbox_recv_irq(mbox->irq_recv_not_empty, mbox); - KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); -} - -/* Start enables the line, starts the jiffy poller and clears the proof; - * stop disables the line, stops the poller and clears the proof. - */ -static void apple_g17_rx_start_stop_test(struct kunit *test) -{ - struct apple_g17_rx_fixture *f = apple_g17_rx_fixture(test); - struct apple_mbox_priv *priv = f->priv; - struct apple_mbox *mbox = &priv->mbox; - unsigned long flags; - bool proven; - - apple_g17_rx_fifo(mbox, false); - priv->rx_irq_proven = true; - apple_mbox_v8_rx_start(mbox); - KUNIT_EXPECT_TRUE(test, priv->rx_irq_live); - KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); - KUNIT_EXPECT_TRUE(test, apple_g17_rx_irq_enabled(mbox)); - - /* Start queued the jiffy poller: it takes a message left queued. */ - apple_g17_rx_fill(f); - KUNIT_EXPECT_TRUE(test, apple_g17_rx_received(f)); - apple_g17_rx_fifo(mbox, false); - - spin_lock_irqsave(&mbox->rx_lock, flags); - apple_mbox_rx_irq_guard(mbox, 1); - proven = priv->rx_irq_proven; - spin_unlock_irqrestore(&mbox->rx_lock, flags); - KUNIT_EXPECT_TRUE(test, proven); - - mbox->active = false; - apple_mbox_v8_rx_stop(mbox); - KUNIT_EXPECT_FALSE(test, priv->rx_irq_live); - KUNIT_EXPECT_FALSE(test, priv->rx_irq_proven); - KUNIT_EXPECT_FALSE(test, apple_g17_rx_irq_enabled(mbox)); - KUNIT_EXPECT_FALSE(test, delayed_work_pending(&priv->rx_poll_work)); -} - static struct kunit_case apple_g17_stall_cases[] = { KUNIT_CASE(apple_g17_rx_flood_bounded_test), KUNIT_CASE(apple_g17_tx_stall_bounded_test), KUNIT_CASE(apple_g17_rx_util_min_test), - KUNIT_CASE(apple_g17_rx_irq_proof_test), - KUNIT_CASE(apple_g17_rx_watchdog_test), - KUNIT_CASE(apple_g17_rx_storm_guard_test), - KUNIT_CASE(apple_g17_rx_start_stop_test), { } }; static struct kunit_suite apple_g17_stall_suite = { diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 879bcb7c826a4..cad52f94fc81f 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -211,16 +210,6 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == #define APPLE_MBOX_TX_RECHECK_MS 2 #define APPLE_G17_MBOX_RX_BUDGET 64 #define APPLE_G17_MBOX_ATOMIC_TX_US 1000 -/* - * T6050 receive until the recv-not-empty interrupt has delivered a message: - * every send arms an hrtimer that polls the receive FIFO at this period, kept - * for this long after the last message or send. A GPU reply lands within one - * 0.22 ms frame; polling every 25 us bounds the added receive latency to ~11% - * of a frame instead of one 4 ms jiffy, while idle receive stays on the jiffy - * poller. - */ -#define APPLE_MBOX_RX_FAST_POLL_US 25 -#define APPLE_MBOX_RX_FAST_GRACE_US 1000 /* One firmware role of an ASCWrap v6 provider, at its fixed CPU-physical windows. */ struct apple_mbox_ascwrap_v6_role_desc { @@ -237,12 +226,6 @@ struct apple_mbox_hw { unsigned int num_ascwrap_v6_roles; /* T8152 GPU ASCWrap v8: wrapper + iop-vbar + post-office window + PSD. */ bool is_ascwrap_v8; - /* - * The recv-not-empty interrupt does not reach the AIC on every board: - * enable it, and also receive by polling until it delivers a message. - * The jiffy poller keeps running after that as a watchdog. - */ - bool rx_poll_until_irq; /* Status word counts RX occupancy and free TX slots; paired transfers. */ bool counted_fifo; bool ap_initializes_mailboxes; @@ -280,21 +263,11 @@ struct apple_mbox_priv { bool rx_poll_only; /* rx_lock: recv interrupt enabled and not yet disabled again. */ bool rx_irq_live; - /* - * rx_lock, read lock-free by the send path: the live recv interrupt - * delivered a message, so fast receive polling stops. Only set while - * rx_irq_live, and cleared with it, or when the jiffy poller finds a - * message the interrupt did not deliver. - */ - bool rx_irq_proven; /* rx_lock: consecutive recv interrupts that drained nothing. */ unsigned int rx_irq_empty_run; /* rx_lock: an unpowered-PSD receive skip was logged this activation. */ bool rx_psd_skip_logged; struct delayed_work rx_poll_work; - /* Kick and fast tick write this lock-free; reads use READ_ONCE. */ - ktime_t rx_fast_until; - struct hrtimer rx_fast_timer; }; static struct apple_mbox_priv *apple_mbox_to_priv(struct apple_mbox *mbox) @@ -305,7 +278,7 @@ static struct apple_mbox_priv *apple_mbox_to_priv(struct apple_mbox *mbox) /* Only providers allocated by apple_mbox_probe() have a receive poller. */ static bool apple_mbox_has_rx_poller(const struct apple_mbox *mbox) { - return mbox->hw->is_ascwrap_v8 || mbox->hw->rx_poll_until_irq; + return mbox->hw->is_ascwrap_v8; } /* Both ASCWrap generations share the consumer lifecycle API below. */ @@ -675,73 +648,6 @@ static struct apple_mbox_msg apple_mbox_read_msg(struct apple_mbox *mbox) return msg; } -static int apple_mbox_poll_locked(struct apple_mbox *mbox); - -/* - * Fast receive tick for rx_poll_until_irq mailboxes (T6050): the same receive - * step apple_mbox_v8_rx_poll_work() runs, under the same rx_lock, with - * apple_mbox_poll_locked() doing the same removal checks. It stops while the - * recv interrupt is proven. Runs in hardirq context. - */ -static enum hrtimer_restart apple_mbox_fast_rx_poll(struct hrtimer *timer) -{ - struct apple_mbox_priv *priv = container_of(timer, struct apple_mbox_priv, - rx_fast_timer); - struct apple_mbox *mbox = &priv->mbox; - unsigned long flags; - ktime_t now; - int consumed = 0; - bool proven; - - if (!READ_ONCE(mbox->active) || READ_ONCE(mbox->removing)) - return HRTIMER_NORESTART; - - spin_lock_irqsave(&mbox->rx_lock, flags); - if (likely(!READ_ONCE(mbox->removing))) - consumed = apple_mbox_poll_locked(mbox); - proven = priv->rx_irq_proven; - spin_unlock_irqrestore(&mbox->rx_lock, flags); - - /* The interrupt receives (see apple_mbox_rx_irq_guard()). */ - if (proven) - return HRTIMER_NORESTART; - - now = ktime_get(); - if (consumed > 0) { - /* Traffic arrived: keep polling as if a send just happened. */ - WRITE_ONCE(priv->rx_fast_until, - ktime_add_us(now, APPLE_MBOX_RX_FAST_GRACE_US)); - } else if (!ktime_before(now, READ_ONCE(priv->rx_fast_until))) { - return HRTIMER_NORESTART; - } - - hrtimer_forward_now(timer, us_to_ktime(APPLE_MBOX_RX_FAST_POLL_US)); - return HRTIMER_RESTART; -} - -/* - * tx_lock held. Lock-free on purpose: apple_mbox_poll_locked() invokes the - * rx callback with rx_lock held, and a callback that sends would then take - * tx_lock under rx_lock. Kick never touches rx_lock, so no ordering exists - * to invert. A kick that loses a race with the timer's final no-restart - * decision leaves the fast poller disarmed once; the next jiffy poll or - * send recovers it. A stale rx_irq_proven costs at most one fast-poll - * burst, or one reply left to the jiffy poller. - */ -static void apple_mbox_fast_rx_kick(struct apple_mbox_priv *priv) -{ - if (READ_ONCE(priv->rx_irq_proven)) - return; - WRITE_ONCE(priv->rx_fast_until, - ktime_add_us(ktime_get(), APPLE_MBOX_RX_FAST_GRACE_US)); - /* Publish the deadline before deciding to arm. */ - smp_mb(); - if (!hrtimer_active(&priv->rx_fast_timer)) - hrtimer_start(&priv->rx_fast_timer, - us_to_ktime(APPLE_MBOX_RX_FAST_POLL_US), - HRTIMER_MODE_REL); -} - int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, bool atomic) { @@ -775,10 +681,9 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, } remaining = deadline - now; if (!atomic && apple_mbox_has_rx_poller(mbox) && - (mbox->hw->rx_poll_until_irq || - apple_mbox_to_priv(mbox)->rx_poll_only)) { + apple_mbox_to_priv(mbox)->rx_poll_only) { /* - * Poll-only v8, or T6050: never enable the send-empty line. + * Poll-only v8: never enable the send-empty line. * Sleep-poll the TX status, touching MMIO only under * tx_lock so that removal cannot unmap it underneath. */ @@ -866,10 +771,6 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, apple_mbox_write_msg(mbox, msg); - /* The reply may be back well within a jiffy: fast-poll for it. */ - if (mbox->hw->rx_poll_until_irq) - apple_mbox_fast_rx_kick(apple_mbox_to_priv(mbox)); - spin_unlock_irqrestore(&mbox->tx_lock, flags); return 0; @@ -989,32 +890,17 @@ int apple_mbox_set_rx_util_min(struct apple_mbox *mbox, int util_min) EXPORT_SYMBOL_GPL(apple_mbox_set_rx_util_min); /* - * Storm guard of the mailboxes with a receive poller, rx_lock held. The - * handler cannot mask the line at the mailbox (no irq controls), so a line - * that keeps firing with nothing to read (for example a send-empty line - * wired as recv-not-empty) would lock up the CPU while always returning - * IRQ_HANDLED. Cut it at the AIC and poll. - * - * A live line that delivers a message has proven itself: fast polling stops - * and the line receives, with the jiffy poller left as a watchdog (see - * apple_mbox_rx_watchdog()). It is level triggered and the G17 handler is - * oneshot, so a message that arrives while the handler runs fires it again. - * If the line later storms, the guard below cuts it and makes sure the jiffy - * poller runs; the next send restarts fast polling. + * ASCWrap v8 storm guard, rx_lock held. The handler cannot mask the line at + * the mailbox (no irq controls), so a line that keeps firing with nothing to + * read (for example a send-empty line wired as recv-not-empty) would lock up + * the CPU while always returning IRQ_HANDLED. Cut it at the AIC and poll. */ -static void apple_mbox_rx_irq_guard(struct apple_mbox *mbox, int drained) +static void apple_mbox_v8_rx_irq_guard(struct apple_mbox *mbox, int drained) { struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); if (drained > 0) { priv->rx_irq_empty_run = 0; - if (priv->rx_irq_live && !priv->rx_irq_proven) { - WRITE_ONCE(priv->rx_irq_proven, true); - if (mbox->hw->rx_poll_until_irq) - dev_dbg(mbox->dev, - "recv-not-empty IRQ %d delivered a message; fast receive polling stopped\n", - mbox->irq_recv_not_empty); - } return; } if (!priv->rx_irq_live || @@ -1022,10 +908,9 @@ static void apple_mbox_rx_irq_guard(struct apple_mbox *mbox, int drained) return; priv->rx_irq_live = false; - WRITE_ONCE(priv->rx_irq_proven, false); disable_irq_nosync(mbox->irq_recv_not_empty); dev_err(mbox->dev, - "recv-not-empty IRQ %d fired %u times without a message; disabled, receive falls back to polling\n", + "ASCWrap v8 recv-not-empty IRQ %d fired %u times without a message; disabled, receive falls back to polling\n", mbox->irq_recv_not_empty, priv->rx_irq_empty_run); queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, 0); } @@ -1043,8 +928,8 @@ static irqreturn_t apple_mbox_recv_irq(int irq, void *data) spin_lock_irqsave(&mbox->rx_lock, flags); if (likely(!READ_ONCE(mbox->removing))) consumed = apple_mbox_poll_locked(mbox); - if (apple_mbox_has_rx_poller(mbox)) - apple_mbox_rx_irq_guard(mbox, consumed); + if (mbox->hw->is_ascwrap_v8) + apple_mbox_v8_rx_irq_guard(mbox, consumed); spin_unlock_irqrestore(&mbox->rx_lock, flags); /* G17 uses a oneshot threaded IRQ. Leave it masked during this sleep @@ -1122,51 +1007,17 @@ int apple_mbox_poll_one(struct apple_mbox *mbox) } EXPORT_SYMBOL_GPL(apple_mbox_poll_one); -/* - * Watchdog after each jiffy poll, rx_lock held. On a line that delivers, the - * interrupt thread has drained the FIFO and the poller finds it empty. A - * message the poller consumed while the interrupt was proven was not - * delivered by the interrupt, or its thread had not run yet: either way - * receive goes back to fast polling on every send until the interrupt - * delivers a message again. All receive steps drain under rx_lock, so - * messages reach the consumer once and in FIFO order whichever step reads - * them. - */ -static void apple_mbox_rx_watchdog(struct apple_mbox *mbox, int consumed) -{ - struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); - - if (consumed <= 0 || !priv->rx_irq_proven) - return; - WRITE_ONCE(priv->rx_irq_proven, false); - dev_dbg(mbox->dev, - "recv-not-empty IRQ %d left a message in the FIFO; receive polling restarted\n", - mbox->irq_recv_not_empty); -} - -/* - * Receive poller: v8 poll-only mode, T6050 for as long as the mailbox is - * active (as a watchdog once the recv interrupt is proven), or after the - * storm guard. - */ +/* ASCWrap v8 receive poller: poll-only mode, or after the storm guard. */ static void apple_mbox_v8_rx_poll_work(struct work_struct *work) { struct apple_mbox_priv *priv = container_of(to_delayed_work(work), struct apple_mbox_priv, rx_poll_work); struct apple_mbox *mbox = &priv->mbox; - unsigned long flags; - int consumed = 0; if (!READ_ONCE(mbox->active) || READ_ONCE(mbox->removing)) return; - - spin_lock_irqsave(&mbox->rx_lock, flags); - if (likely(!READ_ONCE(mbox->removing))) - consumed = apple_mbox_poll_locked(mbox); - apple_mbox_rx_watchdog(mbox, consumed); - spin_unlock_irqrestore(&mbox->rx_lock, flags); - + apple_mbox_poll(mbox); queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, APPLE_ASCWRAP_V8_RX_POLL_JIFFIES); } @@ -1181,13 +1032,11 @@ static void apple_mbox_v8_rx_start(struct apple_mbox *mbox) priv->rx_irq_empty_run = 0; priv->rx_psd_skip_logged = false; priv->rx_irq_live = !priv->rx_poll_only; - /* Each activation proves the line again. */ - WRITE_ONCE(priv->rx_irq_proven, false); spin_unlock_irqrestore(&mbox->rx_lock, flags); - if (priv->rx_poll_only || mbox->hw->rx_poll_until_irq) + if (priv->rx_poll_only) queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, 0); - if (!priv->rx_poll_only) + else enable_irq(mbox->irq_recv_not_empty); } @@ -1206,14 +1055,10 @@ static void apple_mbox_v8_rx_stop(struct apple_mbox *mbox) spin_lock_irqsave(&mbox->rx_lock, flags); live = priv->rx_irq_live; priv->rx_irq_live = false; - WRITE_ONCE(priv->rx_irq_proven, false); spin_unlock_irqrestore(&mbox->rx_lock, flags); if (live) disable_irq(mbox->irq_recv_not_empty); - /* Same cancel-and-drain guarantee for the fast poller as below. */ - if (mbox->hw->rx_poll_until_irq) - hrtimer_cancel(&priv->rx_fast_timer); cancel_delayed_work_sync(&priv->rx_poll_work); } @@ -1400,11 +1245,8 @@ static void apple_mbox_quiesce(void *data) pm_runtime_put_autosuspend(mbox->dev); } /* The v8 poller also stops on the removal fence; make sure it is gone. */ - if (apple_mbox_has_rx_poller(mbox)) { - if (mbox->hw->rx_poll_until_irq) - hrtimer_cancel(&apple_mbox_to_priv(mbox)->rx_fast_timer); + if (apple_mbox_has_rx_poller(mbox)) cancel_delayed_work_sync(&apple_mbox_to_priv(mbox)->rx_poll_work); - } /* Serialize once-only masking with the IRQ and send-error paths. */ spin_lock_irqsave(&mbox->tx_lock, flags); @@ -1620,11 +1462,6 @@ static int apple_mbox_probe(struct platform_device *pdev) return -EINVAL; if (mbox->hw->counted_fifo && !IS_ENABLED(CONFIG_ARM64)) return -EOPNOTSUPP; - if (mbox->hw->rx_poll_until_irq) { - hrtimer_setup(&priv->rx_fast_timer, apple_mbox_fast_rx_poll, - CLOCK_MONOTONIC, HRTIMER_MODE_REL); - INIT_DELAYED_WORK(&priv->rx_poll_work, apple_mbox_v8_rx_poll_work); - } if (mbox->hw->is_ascwrap_v6) { /* @@ -2015,7 +1852,6 @@ static const struct apple_mbox_hw apple_mbox_t6050_ascwrap_v6_hw = { .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, .has_irq_controls = false, - .rx_poll_until_irq = true, }; static const struct apple_mbox_hw apple_mbox_t6050_j775d_ascwrap_v6_hw = { @@ -2032,7 +1868,6 @@ static const struct apple_mbox_hw apple_mbox_t6050_j775d_ascwrap_v6_hw = { .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, .has_irq_controls = false, - .rx_poll_until_irq = true, }; /*