Skip to content

soc: apple: mailbox: arm the T6050 fast receive poller with an atomic flag #1739

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
joshuaswarren wants to merge 1 commit into from
Closed

soc: apple: mailbox: arm the T6050 fast receive poller with an atomic flag #1739

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 101 additions & 0 deletions drivers/soc/apple/mailbox-g17-test.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,111 @@ static void apple_g17_rx_util_min_test(struct kunit *test)
#endif
}

struct apple_g17_fast_rx {
struct apple_mbox_priv priv;
struct completion ticked;
};

static enum hrtimer_restart apple_g17_fast_rx_tick(struct hrtimer *timer)
{
struct apple_g17_fast_rx *f = container_of(timer, struct apple_g17_fast_rx,
priv.rx_fast_timer);

complete(&f->ticked);
return HRTIMER_NORESTART;
}

static struct apple_g17_fast_rx *apple_g17_fast_rx_new(struct kunit *test)
{
struct apple_g17_fast_rx *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL);

KUNIT_ASSERT_NOT_NULL(test, f);
init_completion(&f->ticked);
hrtimer_setup(&f->priv.rx_fast_timer, apple_g17_fast_rx_tick,
CLOCK_MONOTONIC, HRTIMER_MODE_REL);
return f;
}

/* Only the kick that takes rx_fast_armed from 0 to 1 starts the timer. */
static void apple_g17_fast_rx_kick_arms_once_test(struct kunit *test)
{
struct apple_g17_fast_rx *f = apple_g17_fast_rx_new(test);
ktime_t first;

apple_mbox_fast_rx_kick(&f->priv);
KUNIT_EXPECT_EQ(test, atomic_read(&f->priv.rx_fast_armed), 1);
KUNIT_EXPECT_GT(test, wait_for_completion_timeout(&f->ticked, HZ), 0UL);

/* The test tick does not restart, as the real tick does while armed. */
reinit_completion(&f->ticked);
first = f->priv.rx_fast_until;
apple_mbox_fast_rx_kick(&f->priv);
KUNIT_EXPECT_EQ(test, wait_for_completion_timeout(&f->ticked,
msecs_to_jiffies(5)), 0UL);
KUNIT_EXPECT_TRUE(test, ktime_after(f->priv.rx_fast_until, first));

/* The real tick clears the flag on every NORESTART exit; model that. */
atomic_set(&f->priv.rx_fast_armed, 0);
apple_mbox_fast_rx_kick(&f->priv);
KUNIT_EXPECT_GT(test, wait_for_completion_timeout(&f->ticked, HZ), 0UL);
apple_mbox_fast_rx_cancel(&f->priv);
}

/*
* A kick that lands after the tick read its expired deadline but before the
* tick dropped the flag sees the flag set and starts nothing. The tick must
* then notice the new deadline instead of going idle with the kick lost.
*/
static void apple_g17_fast_rx_keeps_racing_kick_test(struct kunit *test)
{
struct apple_g17_fast_rx *f = apple_g17_fast_rx_new(test);
ktime_t before;

/* Deadline passed, no kick: the tick stops and the next kick re-arms. */
atomic_set(&f->priv.rx_fast_armed, 1);
f->priv.rx_fast_until = 0;
KUNIT_EXPECT_FALSE(test, apple_mbox_fast_rx_keep_armed(&f->priv));
KUNIT_EXPECT_EQ(test, atomic_read(&f->priv.rx_fast_armed), 0);

/* The racing kick: the flag is still set, so the kick starts no timer. */
atomic_set(&f->priv.rx_fast_armed, 1);
f->priv.rx_fast_until = 0;
before = ktime_get();
apple_mbox_fast_rx_kick(&f->priv);
KUNIT_EXPECT_TRUE(test, ktime_after(f->priv.rx_fast_until, before));
KUNIT_EXPECT_EQ(test, wait_for_completion_timeout(&f->ticked, 0), 0UL);
/* Pin the deadline live so a slow runner cannot expire the kick's own. */
f->priv.rx_fast_until = ktime_add_ms(ktime_get(), 1000);
KUNIT_EXPECT_TRUE(test, apple_mbox_fast_rx_keep_armed(&f->priv));
KUNIT_EXPECT_EQ(test, atomic_read(&f->priv.rx_fast_armed), 1);
apple_mbox_fast_rx_cancel(&f->priv);
}

/*
* A kick that took the flag but has not called hrtimer_start() yet must keep
* it across a cancel: the timer is not queued, so nothing was cancelled.
*/
static void apple_g17_fast_rx_cancel_keeps_inflight_kick_test(struct kunit *test)
{
struct apple_g17_fast_rx *f = apple_g17_fast_rx_new(test);

atomic_set(&f->priv.rx_fast_armed, 1);
apple_mbox_fast_rx_cancel(&f->priv);
KUNIT_EXPECT_EQ(test, atomic_read(&f->priv.rx_fast_armed), 1);

/* A queued timer is cancelled with its flag, so the next kick re-arms. */
hrtimer_start(&f->priv.rx_fast_timer, ms_to_ktime(1000), HRTIMER_MODE_REL);
apple_mbox_fast_rx_cancel(&f->priv);
KUNIT_EXPECT_EQ(test, atomic_read(&f->priv.rx_fast_armed), 0);
}

static struct kunit_case apple_g17_stall_cases[] = {
KUNIT_CASE(apple_g17_rx_flood_bounded_test),
KUNIT_CASE(apple_g17_tx_stall_bounded_test),
KUNIT_CASE(apple_g17_rx_util_min_test),
KUNIT_CASE(apple_g17_fast_rx_kick_arms_once_test),
KUNIT_CASE(apple_g17_fast_rx_keeps_racing_kick_test),
KUNIT_CASE(apple_g17_fast_rx_cancel_keeps_inflight_kick_test),
{ }
};
static struct kunit_suite apple_g17_stall_suite = {
Expand Down
61 changes: 50 additions & 11 deletions drivers/soc/apple/mailbox.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,14 @@ struct apple_mbox_priv {
/* Kick and fast tick write this lock-free; reads use READ_ONCE. */
ktime_t rx_fast_until;
struct hrtimer rx_fast_timer;
/*
* 1 from the moment a kick takes responsibility for the timer until
* the tick returns without restarting it. The tick forwards its own
* timer only while it holds the flag, and kick starts the timer only
* when it takes the flag, so the two never enqueue the timer at once
* (hrtimer_forward() requires that).
*/
atomic_t rx_fast_armed;
};

static struct apple_mbox_priv *apple_mbox_to_priv(struct apple_mbox *mbox)
Expand Down Expand Up @@ -665,6 +673,20 @@ static struct apple_mbox_msg apple_mbox_read_msg(struct apple_mbox *mbox)

static int apple_mbox_poll_locked(struct apple_mbox *mbox);

/*
* The tick found the deadline passed. Drop the flag, then look at the deadline
* again: a kick between the tick's read and this drop saw the flag set and did
* not start the timer. The full barriers of both xchg() calls pair up, so
* either that kick sees the flag clear and starts the timer, or this recheck
* sees its deadline. True: the tick holds the flag again and must restart.
*/
static bool apple_mbox_fast_rx_keep_armed(struct apple_mbox_priv *priv)
{
atomic_xchg(&priv->rx_fast_armed, 0);
return ktime_before(ktime_get(), READ_ONCE(priv->rx_fast_until)) &&
!atomic_xchg(&priv->rx_fast_armed, 1);
}

/*
* Fast receive tick for rx_poll_only mailboxes (T6050): the same receive
* step apple_mbox_v8_rx_poll_work() runs, under the same rx_lock, with
Expand All @@ -681,8 +703,12 @@ static enum hrtimer_restart apple_mbox_fast_rx_poll(struct hrtimer *timer)
ktime_t now;
int consumed = 0;

if (!READ_ONCE(mbox->active) || READ_ONCE(mbox->removing))
return HRTIMER_NORESTART;
if (!READ_ONCE(mbox->active) || READ_ONCE(mbox->removing)) {
/* A kick that raced the resume must not lose its poll. */
if (!apple_mbox_fast_rx_keep_armed(priv))
return HRTIMER_NORESTART;
goto restart;
}

spin_lock_irqsave(&mbox->rx_lock, flags);
if (likely(!READ_ONCE(mbox->removing)))
Expand All @@ -694,10 +720,12 @@ static enum hrtimer_restart apple_mbox_fast_rx_poll(struct hrtimer *timer)
/* Traffic arrived: keep polling as if a send just happened. */
WRITE_ONCE(priv->rx_fast_until,
ktime_add_us(now, APPLE_MBOX_RX_FAST_GRACE_US));
} else if (!ktime_before(now, READ_ONCE(priv->rx_fast_until))) {
} else if (!ktime_before(now, READ_ONCE(priv->rx_fast_until)) &&
!apple_mbox_fast_rx_keep_armed(priv)) {
return HRTIMER_NORESTART;
}

restart:
hrtimer_forward_now(timer, us_to_ktime(APPLE_MBOX_RX_FAST_POLL_US));
return HRTIMER_RESTART;
}
Expand All @@ -706,22 +734,33 @@ static enum hrtimer_restart apple_mbox_fast_rx_poll(struct hrtimer *timer)
* tx_lock held. Lock-free on purpose: apple_mbox_poll_locked() invokes the
* rx callback with rx_lock held, and a callback that sends would then take
* tx_lock under rx_lock. Kick never touches rx_lock, so no ordering exists
* to invert. A kick that loses a race with the timer's final no-restart
* decision leaves the fast poller disarmed once; the next jiffy poll or
* send recovers it.
* to invert. Only the kick that moves rx_fast_armed from 0 to 1 starts the
* timer, so a kick never enqueues the timer while the tick forwards it.
*/
static void apple_mbox_fast_rx_kick(struct apple_mbox_priv *priv)
{
WRITE_ONCE(priv->rx_fast_until,
ktime_add_us(ktime_get(), APPLE_MBOX_RX_FAST_GRACE_US));
/* Publish the deadline before deciding to arm. */
smp_mb();
if (!hrtimer_active(&priv->rx_fast_timer))
/* The full barrier in xchg() publishes the deadline before the arm. */
if (!atomic_xchg(&priv->rx_fast_armed, 1))
hrtimer_start(&priv->rx_fast_timer,
us_to_ktime(APPLE_MBOX_RX_FAST_POLL_US),
HRTIMER_MODE_REL);
}

/*
* After this the timer is neither queued nor running. The flag is cleared only
* when a queued timer was cancelled: a kick that already took the flag but has
* not called hrtimer_start() yet keeps it, and its timer starts afterwards.
* An idle timer with a stale flag cannot occur, because every tick exit that
* returns NORESTART clears the flag first.
*/
static void apple_mbox_fast_rx_cancel(struct apple_mbox_priv *priv)
{
if (hrtimer_cancel(&priv->rx_fast_timer))
atomic_set(&priv->rx_fast_armed, 0);
}

int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg,
bool atomic)
{
Expand Down Expand Up @@ -1139,7 +1178,7 @@ static void apple_mbox_v8_rx_stop(struct apple_mbox *mbox)
disable_irq(mbox->irq_recv_not_empty);
/* Same cancel-and-drain guarantee for the fast poller as below. */
if (mbox->hw->rx_poll_only)
hrtimer_cancel(&priv->rx_fast_timer);
apple_mbox_fast_rx_cancel(priv);
cancel_delayed_work_sync(&priv->rx_poll_work);
}

Expand Down Expand Up @@ -1328,7 +1367,7 @@ static void apple_mbox_quiesce(void *data)
/* The v8 poller also stops on the removal fence; make sure it is gone. */
if (apple_mbox_has_rx_poller(mbox)) {
if (mbox->hw->rx_poll_only)
hrtimer_cancel(&apple_mbox_to_priv(mbox)->rx_fast_timer);
apple_mbox_fast_rx_cancel(apple_mbox_to_priv(mbox));
cancel_delayed_work_sync(&apple_mbox_to_priv(mbox)->rx_poll_work);
}

Expand Down
Loading