diff --git a/bots/blue-liquidation/README.md b/bots/blue-liquidation/README.md index 66c65dd5..b895a4e7 100644 --- a/bots/blue-liquidation/README.md +++ b/bots/blue-liquidation/README.md @@ -238,7 +238,10 @@ executable quote from the top venue — `0x` / `1inch` / `lifi` (one rate-limite route-bound fixed sell amount; LiFi routes keyless) — falling through to the next venue on failure. A free oracle-based route-quality check (against the full-path oracle reference) rejects any route more than `MAX_ROUTE_IMPACT_BPS` below it. Quotes are made only for the small liquidatable set; a -per-`(id, borrower)` exponential backoff suppresses repeated failures. +per-`(id, borrower)` exponential backoff suppresses repeated failures — including a send the chain +declined with an execution revert, which is a deliberate divergence from `bots/midnight-liquidation` +(blue's liquidation incentive is static, so a shortfall on this block does predict the next one; see +[TIB-2026-08-28](../../docs/decisions/TIB-2026-08-28-midnight-send-shortfall-classification.md)). ### Simulation diff --git a/bots/blue-liquidation/src/index.ts b/bots/blue-liquidation/src/index.ts index e146e247..c18b3328 100644 --- a/bots/blue-liquidation/src/index.ts +++ b/bots/blue-liquidation/src/index.ts @@ -38,10 +38,10 @@ import { SETTLED_COOLDOWN_BLOCKS } from './constants' import { createGraphqlCandidateSource, discoverCandidates } from './discovery/borrowers' import { encodeLiquidationExec } from './execution/encode-call' import { composeQuoting } from './quotes' +import { revertReason } from './revert.utils' import { runTick } from './runner/tick' import { readBlueLiquidationLens } from './state/lens.sol' import { createMarketParamsResolver, multicallIdToMarketParams } from './state/market-params' -import { revertReason } from './tx-error' async function main() { const config = loadConfig() diff --git a/bots/blue-liquidation/src/tx-error.ts b/bots/blue-liquidation/src/revert.utils.ts similarity index 100% rename from bots/blue-liquidation/src/tx-error.ts rename to bots/blue-liquidation/src/revert.utils.ts diff --git a/bots/blue-liquidation/src/runner/tick.ts b/bots/blue-liquidation/src/runner/tick.ts index 4d782e8d..9fb332ff 100644 --- a/bots/blue-liquidation/src/runner/tick.ts +++ b/bots/blue-liquidation/src/runner/tick.ts @@ -226,6 +226,9 @@ export async function runTick(deps: { // block from re-quoting, re-simulating and re-sending it — reaching this line at all means any // earlier entry had already expired, so leaving it untouched suppresses nothing. A queue-wide // refusal says nothing about the position, so it records nothing. + // Blue keeps backoff on every rejected send, including an execution revert, because its + // liquidation incentive is static — unlike midnight, which exempts that case. See + // docs/decisions/TIB-2026-08-28-midnight-send-shortfall-classification.md. if (outcome.reason === 'send_failed') backoff.record(label, chainHead) } } diff --git a/bots/blue-liquidation/test/runner/tick.test.ts b/bots/blue-liquidation/test/runner/tick.test.ts index b0867952..bb14c184 100644 --- a/bots/blue-liquidation/test/runner/tick.test.ts +++ b/bots/blue-liquidation/test/runner/tick.test.ts @@ -349,7 +349,18 @@ describe('runTick', () => { // would re-quote, re-simulate and re-send. const { counters, backoff } = await runWith({ seedBackoffAt: 1n, - submitOutcome: { sent: false, reason: 'send_failed' } + submitOutcome: { sent: false, reason: 'send_failed', executionRevert: false } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) + expect(backoff.shouldSkip(LABEL, 101n)).toBe(true) + }) + + it("keeps backoff on an execution-reverted send: blue's incentive is static", async () => { + // Pins the divergence documented at the backoff.record call in src/runner/tick.ts: midnight + // exempts this case, blue must not. + const { counters, backoff } = await runWith({ + seedBackoffAt: 1n, + submitOutcome: { sent: false, reason: 'send_failed', executionRevert: true } }) expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) expect(backoff.shouldSkip(LABEL, 101n)).toBe(true) diff --git a/bots/midnight-liquidation/README.md b/bots/midnight-liquidation/README.md index c9773ee6..60fd3f47 100644 --- a/bots/midnight-liquidation/README.md +++ b/bots/midnight-liquidation/README.md @@ -97,7 +97,7 @@ Environment variables: | `HEADROOM_FLOOR_BPS` | no | `3` | **Lower bound** on swap execution cost — the cheapest route you would ever expect, NOT a typical cost. A seize-exact plan's whole margin is the incentive `(lif - 1)/lif`, so a plan below this floor cannot fund its own repay by any route and is skipped as `plan.skipped` / `insufficient_headroom` before it costs a quote, a simulation or a gas estimate. Post-maturity the incentive ramps from zero over an hour, so this acts as a pure time gate: `3` suppresses roughly the first 25s on a 4.4%-maxLif tier. Set it too high and it blinds the earliest, most contested part of a maturity. `0` disables the gate. | | `MIN_SURPLUS_BPS` | no | `0` | Surplus over break-even a quoted route's **expected** output must clear before the bot spends a simulation on it, in bps of the plan's contract-derived repay. `0` is pure break-even: both sides then come from the contract's own formula with no tuned value, so the gate can only reject plans that would have reverted anyway. It gates the expected output only — the min-out actually encoded in the swap calldata stays at break-even — so raising it buys margin against a route that underperforms its quote, not against oracle drift between simulation and inclusion. | | `PENDLE_SLIPPAGE_BPS` | no | `50` | Slippage for the Pendle PT → underlying unwrap hop (before the downstream venue sells). | -| `BACKOFF_BASE_BLOCKS` / `BACKOFF_MAX_BLOCKS` | no | `2` / `64` | Exponential per-position cooldown (in blocks) after a failed quote/simulate, bounding API + RPC usage under a backlog. | +| `BACKOFF_BASE_BLOCKS` / `BACKOFF_MAX_BLOCKS` | no | `2` / `64` | Exponential per-position cooldown (in blocks) after a failed quote/simulate, bounding API + RPC usage under a backlog. An economic refusal (`floor_unmet`, an unprofitable quote) never arms it, and neither does a send the chain itself declined — see [Broadcast And Pending Queue](#broadcast-and-pending-queue). | | `POSITION_LIQUIDATION_COOLDOWN_MS` | no | `0` | Opt-in per-position cooldown (ms) after a failed liquidation attempt; `0` disables it (re-attempt every tick). | | `BETTERSTACK_SOURCE_TOKEN` / `BETTERSTACK_INGESTING_HOST` | no | — | Opt-in log shipping; when both are set the bot's in-process loglayer transport ships structured logs to BetterStack (inert otherwise). | | `BETTERSTACK_HEARTBEAT_URL` | no | — | Optional Better Stack Uptime heartbeat URL, pinged every minute; failures only log a warning and never interrupt liquidations. | @@ -511,7 +511,7 @@ The bot computes the oracle-priced reference output for free (no extra API call) route more than `MAX_ROUTE_IMPACT_BPS` below it (`quote.route_quality_failed`). Quote failures (no route, timeout, rate-limited, API error) log `quote.failed`; once every ranked venue is exhausted the position is backed off — an exponential per-position cooldown that bounds API + RPC usage when many -positions fail (the rate-limit defense). A successful submit clears the backoff. +positions fail (the rate-limit defense). A successful submit clears the backoff; nothing else does. If no venue is enabled (bad-debt-only mode) or the collateral is on `EXCLUDE_COLLATERALS`, the tick logs `config.no_swap_path` and skips the candidate (no API call, no backoff). Pure bad-debt @@ -547,6 +547,42 @@ While a label is pending, later ticks skip that position. On each block the queu logs confirmed or reverted transactions, and fee-bumps stuck transactions until either they confirm, hit the fee ceiling, or exhaust bump attempts. +A queue answer that broadcast nothing is classified three ways, counted on `tick.end` as +`sendRefused` / `sendReverted` / `sendRejected` (which sum to `notSent`): + +- **refused** — the queue declined before reaching the send (aborted-send latch, failed nonce sync, + nonce hole). Queue-wide, so it is held against no position. +- **execution-reverted** — the node rejected this position's own transaction with an on-chain + execution revert (`tx.submit_failed`, `executionRevert: true`, plus the 4-byte `selector` when the + payload carried one). **This does not extend the position's suppression window**, and it also + exempts the position from an entry a sibling candidate armed, whichever order the two ran in — an + execution revert, unlike a broadcast, does not stop the next-ranked sibling from being tried. + Post-maturity the LIF ramps over an hour, so a min-out shortfall says nothing about the next + attempt; backing off sampled that ramp + exponentially, which is what turned a maturity into 4–9 minutes per position on 2026-08-28. The + position is instead retried as fast as the tick can drain, with skipped heights coalesced. An + opted-in `POSITION_LIQUIDATION_COOLDOWN_MS` window is deliberately left armed: it is a flat + operator throttle, not a ramp sampler, so lifting it is an operator's call. +- **rejected** — the send machinery failed (nonce, funds, RPC). Nothing was learned about the plan, + so the position backs off exactly as before. + +Because the execution-reverted case carries no throttle at all, an unbroken streak of +execution-reverted sends on one position is tracked and reported as `send.revert_streak` (warn) on the +one send that first takes it past 15 minutes, carrying the revert count, the streak duration, the last +selector, and whether that selector stayed constant across the streak. Only that crossing is logged: +with no throttle on the path, warning on every later revert would ship a line per tick — two, when +both of a position's siblings revert — for as long as it stays stuck. A constant selector over a long +streak points at a structural fault — an expired route deadline, malformed aggregator calldata, +an estimator/provider discrepancy, a gate that keeps closing — rather than an incentive that has yet +to catch up. The threshold is a duration and not an attempt count on purpose: the incentive ramps on +wall clock, so attempts-to-clear is `clearing_time / sweep_period` and shrinks every time the bot +gets faster, which a count threshold would have to be recalibrated against. The streak only reports; +it never suppresses. + +`bots/blue-liquidation` deliberately diverges here and keeps backoff on every rejected send, +execution reverts included: its liquidation incentive is static rather than ramping, so a declined +send there really is evidence about the next block. + Queue state is in-memory. On restart, chain truth wins: the bot rediscovers live candidates and the signer nonce cursor starts from the pending chain nonce. If the initial raw broadcast fails after a nonce is claimed but before a hash is returned, the signer rolls the cursor back and the queue aborts diff --git a/bots/midnight-liquidation/src/index.ts b/bots/midnight-liquidation/src/index.ts index 6ca9e564..8795b135 100644 --- a/bots/midnight-liquidation/src/index.ts +++ b/bots/midnight-liquidation/src/index.ts @@ -48,9 +48,10 @@ import { createListedMarketFilter, createUnionListedMarketFilter } from './disco import { createTokenPriceSource } from './discovery/token-prices' import { encodeLiquidationExec } from './execution/encode-call' import { composeQuoting } from './quotes' +import { revertReason } from './revert.utils' +import { createRevertStreakStore } from './runner/revert-streak' import { runTick } from './runner/tick' import { readMidnightLiquidationLens } from './state/lens.sol' -import { revertReason } from './tx-error' async function main() { const config = loadConfig() @@ -220,6 +221,9 @@ async function main() { // Opt-in per-position cooldown (default disabled): one in-memory store for the process lifetime, // complementary to `backoff` (see POSITION_LIQUIDATION_COOLDOWN_MS). const cooldown = createCooldownStore({ cooldownMs: config.positionCooldownMs }) + // Telemetry only: an execution-reverted send is exempt from backoff, so this is what reports a + // position whose sends keep being declined — see `createRevertStreakStore` for the threshold. + const revertStreaks = createRevertStreakStore() // The exec calldata for one liquidation — the same bytes the simulate gate checks and the queue // broadcasts, so a sim-ok plan and its broadcast can't drift. @@ -372,6 +376,7 @@ async function main() { }, backoff, cooldown, + revertStreaks, inflightLabels: () => queue.inflightLabels(), usdValueOf: tokenPrices.usdValueOf, // Phase A.5's probe seam. `refresh` is staleness-gated and `select` is a pure cache lookup, so diff --git a/bots/midnight-liquidation/src/tx-error.ts b/bots/midnight-liquidation/src/revert.utils.ts similarity index 100% rename from bots/midnight-liquidation/src/tx-error.ts rename to bots/midnight-liquidation/src/revert.utils.ts diff --git a/bots/midnight-liquidation/src/runner/revert-streak.ts b/bots/midnight-liquidation/src/runner/revert-streak.ts new file mode 100644 index 00000000..8e1ceb52 --- /dev/null +++ b/bots/midnight-liquidation/src/runner/revert-streak.ts @@ -0,0 +1,101 @@ +import type { Hex } from 'viem' + +/** + * How long an unbroken execution-revert streak may run before {@link RevertStreak.escalate} reports it. + * + * A DURATION rather than an attempt count, because post-maturity LIF ramps on wall-clock: a position + * becomes fundable at a moment, not after N tries, and attempts-to-clear is `clearing_time / + * sweep_period` — the sweep period being exactly what shrinks as the bot gets faster. A count + * threshold would need recalibrating on every latency win and would start firing on healthy positions. + * 15 minutes sits past the 4–9 minutes positions actually took to clear on 2026-08-28 and well inside + * the 60-minute `TIME_TO_MAX_LIF` ramp. + */ +export const REVERT_STREAK_ESCALATE_MS = 15 * 60_000 + +/** What {@link RevertStreakStore.record} learned about the streak the just-recorded revert extends. */ +export type RevertStreak = { + /** Consecutive execution-reverted sends, this one included. */ + count: number + /** Wall-clock ms from the streak's FIRST execution-reverted send to this one; `0` on the first. */ + durationMs: number + /** The 4-byte selector this send reverted with, absent when the payload carried none. */ + selector: Hex | undefined + /** + * True while every send in the streak reported the same selector — much stronger evidence of a + * structural fault (a closed gate, malformed calldata, an estimator discrepancy) than a mixed + * streak, which reads as ordinary min-out shortfalls against whichever pool the route hit. + */ + selectorConstant: boolean + /** + * Where this revert sits against the store's threshold ({@link REVERT_STREAK_ESCALATE_MS}): + * `crossed` on the one revert that first runs past it, `ongoing` on every revert after that. A + * reporter must fire on `crossed` alone — `ongoing` repeats for as long as the position stays + * stuck, which on a per-block sweep is unbounded. + */ + escalate: 'below' | 'crossed' | 'ongoing' +} + +/** + * Per-position tracker of consecutive execution-reverted sends, keyed by the `${id}:${borrower}` + * label. The backstop that makes running an execution-reverted send with NO retry throttle + * defensible: a min-out shortfall clears as the LIF ramps, but a persistent estimator-only failure — + * an expired route deadline, malformed aggregator calldata, a gate that keeps closing — can pass the + * simulation and fail the send's gas estimate indefinitely, which would otherwise burn quotes forever + * without progressing. Both calls run at `latest`; they diverge because they are issued by DIFFERENT + * clients over their own `failover` transport pairs, so they can observe different heads, different + * provider-side estimator behaviour, and pool state that moved in between. It only reports; it never + * suppresses. + * + * In-memory only, like the shared `Backoff` and `CooldownStore` — chain truth wins on restart. Entries + * for a position that recovers to non-liquidatable are never re-checked and linger until process exit: + * the same accepted, bounded leak `createBackoff` documents at its canonical home. + */ +export type RevertStreakStore = { + /** Extends `label`'s streak with one execution-reverted send and returns its state. */ + record: (label: string, selector?: Hex) => RevertStreak + /** Ends `label`'s streak — a broadcast, or a send failure the chain did not decline. */ + reset: (label: string) => void +} + +type Entry = { + count: number + startedAt: number + selector: Hex | undefined + constant: boolean + escalated: boolean +} + +/** + * Lives in this bot rather than `@repo/bot-kit` because the threshold is calibrated against one bot's + * incentive shape — a wall-clock LIF ramp — and no second consumer exists yet. + */ +export const createRevertStreakStore = ( + opts: { escalateAfterMs?: number; now?: () => number } = {} +): RevertStreakStore => { + const escalateAfterMs = opts.escalateAfterMs ?? REVERT_STREAK_ESCALATE_MS + const now = opts.now ?? (() => Date.now()) + const streaks = new Map() + + return { + record: (label, selector) => { + const at = now() + const previous = streaks.get(label) + const count = (previous?.count ?? 0) + 1 + const startedAt = previous?.startedAt ?? at + const constant = previous ? previous.constant && previous.selector === selector : true + const durationMs = at - startedAt + const past = durationMs > escalateAfterMs + streaks.set(label, { count, startedAt, selector, constant, escalated: past }) + return { + count, + durationMs, + selector, + selectorConstant: constant, + escalate: past ? (previous?.escalated ? 'ongoing' : 'crossed') : 'below' + } + }, + reset: label => { + streaks.delete(label) + } + } +} diff --git a/bots/midnight-liquidation/src/runner/tick.ts b/bots/midnight-liquidation/src/runner/tick.ts index c5d4b6fc..f123ab44 100644 --- a/bots/midnight-liquidation/src/runner/tick.ts +++ b/bots/midnight-liquidation/src/runner/tick.ts @@ -10,6 +10,7 @@ import type { BorrowerCandidate } from '../discovery/borrowers' import type { Market } from '../execution/encode-call' import type { LiquidationPlan, PlanSkipReason } from '../sizing/plan' import type { LensInput, LensOut } from '../state/lens.sol' +import type { RevertStreakStore } from './revert-streak' import { MAX_PRESELECTED_CANDIDATES_PER_POSITION } from '../constants' import { USD_PRICE_SCALE_DECIMALS } from '../discovery/token-prices' @@ -37,6 +38,7 @@ import { rankByNetUsdSurplus, scoreNetOfRouteCost } from './ranking' * candidates === cooledDown + backoffSkipped + siblingSkipped + preselectSkipped * + noSwapPath + quoteFailed + quoteUnprofitable + ok + reverted (per CANDIDATE) * ok === submitted + notSent + * notSent === sendRefused + sendReverted + sendRejected * ``` * * **The two middle identities count different things**, because one position can yield several @@ -50,8 +52,8 @@ import { rankByNetUsdSurplus, scoreNetOfRouteCost } from './ranking' * `candidates` sum, and any future plan-stage skip rides `planSkipped`. `unpriced` is an attribute: an * unpriced candidate is still worked, just ordered last, so it deliberately joins no sum. * - * On `complete: false` the last identity is short by one: an aborting `submit` throws after `ok` was - * counted. + * On `complete: false` the `ok` identity is short by one: an aborting `submit` throws after `ok` was + * counted. The `notSent` decomposition still holds, since that throw increments none of its four terms. */ type TickCounters = { /** Lens inputs read this tick — the post-whitelist discovery universe. */ @@ -129,6 +131,25 @@ type TickCounters = { submitted: number /** The queue returned without broadcasting (a send failure, or a queue-wide refusal). */ notSent: number + /** + * A queue-wide refusal (aborted send latch, failed nonce sync, nonce hole) that would have refused + * any position, so it is held against none — see {@link SubmitOutcome}. + */ + sendRefused: number + /** + * The node declined THIS plan with an on-chain execution revert. An economic outcome post-maturity, + * counted separately from {@link TickCounters.sendRejected} because it deliberately does not extend + * the position's suppression window: every one observed on 2026-08-28 was a min-out shortfall against + * whichever pool the aggregator routed through, and the LIF ramp lifts break-even on a wall-clock + * scale, so it says nothing about the next block. + * + * **Not folded into `quoteUnprofitable`**, however similar the verdict reads: that counter sits in the + * per-candidate `candidates` sum and this candidate was already counted there as `ok`, so sharing it + * would break that identity by one for every execution-reverted send. + */ + sendReverted: number + /** The send machinery failed (nonce, funds, RPC) — nothing was learned about the plan, so it backs off. */ + sendRejected: number /** * Planned candidates whose loan token had no usable USD price, so they were ordered last rather than * ranked. An attribute of a worked position, not a loop exit — it joins no identity. A persistently @@ -535,9 +556,10 @@ export async function runTick(deps: { /** * Broadcasts a plan via the pending queue (builds the exec tx, derives fees, tracks the nonce). * Resolves whether a transaction actually went out: ONLY `sent: true` may clear the - * position's backoff or count as `submitted`. A `sent: false` outcome carries why, and the two - * reasons are not interchangeable — see {@link SubmitOutcome}. Throws when a send claimed a nonce but produced no hash — the tick aborts - * by design, so the signer's cursor rollback is not raced. + * position's backoff or count as `submitted`. A `sent: false` outcome carries why, and the three + * cases are not interchangeable — see {@link SubmitOutcome} and the submit branch below. Throws when + * a send claimed a nonce but produced no hash — the tick aborts by design, so the signer's cursor + * rollback is not raced. */ submit: (args: { market: Market @@ -555,6 +577,11 @@ export async function runTick(deps: { * included). Disabled by default (`POSITION_LIQUIDATION_COOLDOWN_MS=0`) — `shouldSkip` always false. */ cooldown: CooldownStore + /** + * Watches consecutive execution-reverted sends per position and reports a streak that has run too + * long. Pure telemetry — it suppresses nothing (see {@link RevertStreakStore}). + */ + revertStreaks: RevertStreakStore /** Labels (`${id}:${borrower}`) already in flight — skipped to avoid re-submitting each block. */ inflightLabels: () => ReadonlySet /** @@ -584,6 +611,7 @@ export async function runTick(deps: { submit, backoff, cooldown, + revertStreaks, inflightLabels, usdValueOf, routing, @@ -624,6 +652,9 @@ export async function runTick(deps: { reverted: 0, submitted: 0, notSent: 0, + sendRefused: 0, + sendReverted: 0, + sendRejected: 0, unpriced: 0 } @@ -704,6 +735,9 @@ export async function runTick(deps: { // being grouped to make the bookkeeping work. const submittedLabels = new Set() const pendingBackoff = new Set() + // Positions an execution-reverted send exempts from `pendingBackoff` — see the `executionRevert` + // arm for why the exemption has to be a latch. + const backoffExempt = new Set() const pendingCooldown = new Set() let complete = false // Firm venue calls this tick, `null` while no quote reported any: an absent `firmCalls` is unknown, @@ -897,20 +931,56 @@ export async function runTick(deps: { if (outcome.sent) { submittedLabels.add(label) backoff.clear(label) + revertStreaks.reset(label) // A broadcast settles the position for this tick, so a lower-ranked sibling's earlier // failure must not still suppress it: this candidate succeeded where that one didn't. pendingBackoff.delete(label) pendingCooldown.delete(label) counters.submitted += 1 } else { - // Nothing was broadcast, so the position's failure history stands: clearing backoff here is - // what let a failing position reset to attempt 1 and re-quote every other block. + // Nothing was broadcast, so the position's failure history stands: `backoff.clear` stays gated + // on a real broadcast, because clearing it here is what let a failing position reset to attempt + // 1 and re-quote every other block. counters.notSent += 1 - // A rejected send is a fact about THIS position, and backoff is the only thing stopping the next - // block from re-quoting, re-simulating and re-sending it — reaching this line at all means any - // earlier entry had already expired, so leaving it untouched suppresses nothing. A queue-wide - // refusal says nothing about the position, so it records nothing. - if (outcome.reason === 'send_failed') pendingBackoff.add(label) + if (outcome.reason === 'refused') { + // Queue-wide — it would have refused any position. So it arms nothing, and it does not break + // the revert streak either: nothing was sent, so the chain said nothing about this plan. + counters.sendRefused += 1 + } else if (outcome.executionRevert) { + counters.sendReverted += 1 + // The chain declined this plan at this block. Post-maturity that is an economic verdict on a + // ramping incentive, not a fact about the next block, so it must not extend the window. + // A LATCH, not a delete, because `pendingBackoff` is keyed by POSITION and the exemption has + // to hold whatever order this position's siblings ran in: unlike a broadcast, an execution + // revert does not enter `submittedLabels`, so the next-ranked candidate still runs and can + // arm the entry AFTER this one — a delete would only survive when the reverted send happened + // to be the position's last event of the tick. `pendingCooldown` is deliberately left armed: + // it is a flat, default-off window an operator opts into to throttle a position class, so + // lifting it is an operator's call rather than an inference from one sibling's outcome. + backoffExempt.add(label) + // No throttle on this path, so a persistent estimator-only revert (expired route deadline, + // malformed calldata, a gate that keeps closing) would re-quote forever without progressing. + // Reported, never suppressed — see RevertStreakStore. + const streak = revertStreaks.record(label, outcome.selector) + // Only the crossing, so one stuck position is one warn per streak rather than one per tick + // (two, when both its siblings revert) for as long as it stays stuck. + if (streak.escalate === 'crossed') { + logger.warn('send.revert_streak', { + marketId: pair.id, + borrower: pair.borrower, + reverts: streak.count, + durationMs: streak.durationMs, + selector: streak.selector ?? null, + selectorConstant: streak.selectorConstant + }) + } + } else { + counters.sendRejected += 1 + // The send machinery failed (nonce, funds, RPC): a fact about this position that says nothing + // about the chain's view of the plan, so it arms backoff and ends the streak. + pendingBackoff.add(label) + revertStreaks.reset(label) + } } } } @@ -919,7 +989,9 @@ export async function runTick(deps: { // Suppression applied once per position, after every candidate has had its turn (see the phase B // preamble). In the `finally` so an aborting `submit` still records what the tick learned. for (const label of pendingCooldown) cooldown.mark(label) - for (const label of pendingBackoff) backoff.record(label, chainHead) + for (const label of pendingBackoff) { + if (!backoffExempt.has(label)) backoff.record(label, chainHead) + } logger.info('tick.end', { ...counters, firmCalls, diff --git a/bots/midnight-liquidation/test/runner/revert-streak.test.ts b/bots/midnight-liquidation/test/runner/revert-streak.test.ts new file mode 100644 index 00000000..625ef02f --- /dev/null +++ b/bots/midnight-liquidation/test/runner/revert-streak.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from 'vitest' + +import { createRevertStreakStore, REVERT_STREAK_ESCALATE_MS } from '../../src/runner/revert-streak' + +const LABEL = 'market:borrower' +const OTHER = 'market:other' +const SHORTFALL = '0x08c379a0' as const +const PANIC = '0x4e487b71' as const + +// A controllable clock, since the threshold is a duration — the whole point of the store. +const clock = (start = 1_000_000) => { + let at = start + return { now: () => at, advance: (ms: number) => (at += ms) } +} + +describe('createRevertStreakStore', () => { + it('accumulates consecutive reverts and reports the streak duration', () => { + const time = clock() + const store = createRevertStreakStore({ now: time.now }) + expect(store.record(LABEL, SHORTFALL)).toStrictEqual({ + count: 1, + durationMs: 0, + selector: SHORTFALL, + selectorConstant: true, + escalate: 'below' + }) + time.advance(30_000) + expect(store.record(LABEL, SHORTFALL)).toStrictEqual({ + count: 2, + durationMs: 30_000, + selector: SHORTFALL, + selectorConstant: true, + escalate: 'below' + }) + time.advance(45_000) + // Measured from the FIRST revert in the streak, not from the previous one. + expect(store.record(LABEL, SHORTFALL)).toStrictEqual({ + count: 3, + durationMs: 75_000, + selector: SHORTFALL, + selectorConstant: true, + escalate: 'below' + }) + }) + + it('escalates only once the streak has run past the threshold', () => { + const time = clock() + const store = createRevertStreakStore({ escalateAfterMs: 1000, now: time.now }) + expect(store.record(LABEL).escalate).toBe('below') + time.advance(1000) + // Strictly past, so a streak exactly at the threshold is not yet escalated. + expect(store.record(LABEL).escalate).toBe('below') + time.advance(1) + expect(store.record(LABEL).escalate).toBe('crossed') + }) + + it('reports the crossing once, then ongoing, so a stuck position is one warn', () => { + const time = clock() + const store = createRevertStreakStore({ escalateAfterMs: 1000, now: time.now }) + store.record(LABEL) + time.advance(1001) + expect(store.record(LABEL).escalate).toBe('crossed') + // Same tick, second sibling: the crossing is already spent. + expect(store.record(LABEL).escalate).toBe('ongoing') + time.advance(60_000) + expect(store.record(LABEL).escalate).toBe('ongoing') + }) + + it('defaults to a 15-minute threshold', () => { + const time = clock() + const store = createRevertStreakStore({ now: time.now }) + store.record(LABEL) + time.advance(REVERT_STREAK_ESCALATE_MS) + expect(store.record(LABEL).escalate).toBe('below') + time.advance(1) + expect(store.record(LABEL).escalate).toBe('crossed') + }) + + it('starts a fresh streak after a reset', () => { + const time = clock() + const store = createRevertStreakStore({ escalateAfterMs: 1000, now: time.now }) + store.record(LABEL) + time.advance(5000) + store.record(LABEL) + store.reset(LABEL) + // A broadcast, or a send failure the chain did not decline, ends the streak: the next revert is + // attempt one at zero duration, so it cannot inherit an escalation from before the reset. + expect(store.record(LABEL)).toStrictEqual({ + count: 1, + durationMs: 0, + selector: undefined, + selectorConstant: true, + escalate: 'below' + }) + // And the fresh streak can cross again, rather than the spent crossing latching forever. + time.advance(1001) + expect(store.record(LABEL).escalate).toBe('crossed') + }) + + it('reports a constant selector, and stops once one differs', () => { + const store = createRevertStreakStore({ now: clock().now }) + expect(store.record(LABEL, SHORTFALL).selectorConstant).toBe(true) + expect(store.record(LABEL, SHORTFALL)).toStrictEqual({ + count: 2, + durationMs: 0, + selector: SHORTFALL, + selectorConstant: true, + escalate: 'below' + }) + // A mixed streak reads as ordinary min-out shortfalls against different pools; once mixed it stays + // mixed, so a later repeat cannot re-assert a structural fault. + expect(store.record(LABEL, PANIC)).toStrictEqual({ + count: 3, + durationMs: 0, + selector: PANIC, + selectorConstant: false, + escalate: 'below' + }) + expect(store.record(LABEL, PANIC).selectorConstant).toBe(false) + }) + + it('treats a missing selector as its own value', () => { + const store = createRevertStreakStore({ now: clock().now }) + expect(store.record(LABEL)).toStrictEqual({ + count: 1, + durationMs: 0, + selector: undefined, + selectorConstant: true, + escalate: 'below' + }) + expect(store.record(LABEL).selectorConstant).toBe(true) + expect(store.record(LABEL, SHORTFALL).selectorConstant).toBe(false) + }) + + it('keeps streaks independent per position', () => { + const time = clock() + const store = createRevertStreakStore({ escalateAfterMs: 1000, now: time.now }) + store.record(LABEL) + time.advance(2000) + store.record(OTHER) + expect(store.record(LABEL).escalate).toBe('crossed') + expect(store.record(OTHER).escalate).toBe('below') + store.reset(LABEL) + expect(store.record(OTHER).count).toBe(3) + }) +}) diff --git a/bots/midnight-liquidation/test/runner/tick.test.ts b/bots/midnight-liquidation/test/runner/tick.test.ts index 14c61edd..1232cfec 100644 --- a/bots/midnight-liquidation/test/runner/tick.test.ts +++ b/bots/midnight-liquidation/test/runner/tick.test.ts @@ -9,6 +9,7 @@ import { getAddress } from 'viem' import { describe, expect, it } from 'vitest' import type { BorrowerCandidate } from '../../src/discovery/borrowers' +import type { RevertStreak, RevertStreakStore } from '../../src/runner/revert-streak' import type { LiquidationPlan } from '../../src/sizing/plan' import type { LensCollateral, LensInput, LensOut } from '../../src/state/lens.sol' @@ -27,6 +28,16 @@ function spyLogger() { return { logger, events } } +// The tick must classify no send outcome on a path that aborts, so either call is a failure. +const unreachableStreaks: RevertStreakStore = { + record: () => { + throw new Error('unexpected streak record') + }, + reset: () => { + throw new Error('unexpected streak reset') + } +} + // The sums documented on TickCounters. Asserted rather than eyeballed so a new loop exit that forgets // its counter fails a test instead of silently dropping a position from the tally. const expectCounterIdentities = (c: Record) => { @@ -48,6 +59,9 @@ const expectCounterIdentities = (c: Record) => { c.reverted! ) expect(c.ok).toBe(c.submitted! + c.notSent!) + // Decomposes `notSent`, so it sits after that sum: the three no-broadcast outcomes are handled + // differently and must never be collapsed. + expect(c.notSent).toBe(c.sendRefused! + c.sendReverted! + c.sendRejected!) } const BORROWER: Address = getAddress('0x1111111111111111111111111111111111111111') @@ -144,6 +158,14 @@ function lensOut(overrides: Partial = {}): LensOut { const candidates = (...borrowers: Address[]): BorrowerCandidate[] => borrowers.map(borrower => ({ marketId: MARKET, borrower })) +// A two-slot position: `collateralParams[1]` (COLLATERAL, 86% lltv) needs a swap and outranks +// `collateralParams[0]` (the loan token itself, 98% lltv) on surplus, so it is tried first. +const twoSlots = () => + lensOut({ + activatedBitmap: 0b11n, + collaterals: [slot({ index: 1 }), slot({ index: 0, maxLif: LOAN_MAX_LIF })] + }) + // Per-borrower readings, so one tick can hold candidates with different surpluses / loan tokens. function stubReadLensByBorrower(byBorrower: Map) { return async (pairs: LensInput[]) => { @@ -177,10 +199,12 @@ function runWith(opts: { headroomFloorBps?: number minSurplusBps?: number cooldown?: CooldownStore - /** Models the queue's outcome; the two no-broadcast reasons are NOT interchangeable. */ + /** Models the queue's outcome; the three no-broadcast outcomes are NOT interchangeable. */ submitOutcome?: SubmitOutcome /** Models a send that claimed a nonce but produced no hash, which aborts the tick. */ submitThrows?: Error + /** What the streak store reports back for a recorded execution revert. */ + revertStreak?: RevertStreak /** Distinct readings per borrower, for ordering cases. Overrides `out`. */ outsByBorrower?: Map /** Defaults to an identity valuation, so surplusUsd tracks surplus and ordering is deterministic. */ @@ -218,6 +242,26 @@ function runWith(opts: { // Collateral tokens phase A.5 asked to be warmed, in call order — one entry per refresh actually // issued, so a duplicate here is a duplicated probe burst. const warmed: Address[] = [] + // Every streak call in order, so a test can pin that the tick records the chain's declines and ends + // the streak on everything else — the store's own arithmetic is covered in revert-streak.test.ts. + const streakCalls: { kind: 'record' | 'reset'; label: string; selector?: Hex }[] = [] + const revertStreaks: RevertStreakStore = { + record: (label, selector) => { + streakCalls.push({ kind: 'record', label, selector }) + return ( + opts.revertStreak ?? { + count: 1, + durationMs: 0, + selector, + selectorConstant: true, + escalate: 'below' + } + ) + }, + reset: label => { + streakCalls.push({ kind: 'reset', label }) + } + } const routing = { resolveRoute: async (plan: LiquidationPlan, out: LensOut) => { const collateral = out.market.collateralParams[plan.collateralIndex] @@ -279,6 +323,7 @@ function runWith(opts: { }, backoff, cooldown, + revertStreaks, inflightLabels: () => opts.inflight ?? new Set(), usdValueOf: opts.usdValueOf ?? ((_loanToken, loanUnits) => loanUnits), routing, @@ -293,6 +338,7 @@ function runWith(opts: { quoteCalls: () => quoteCalls, order, warmed, + streakCalls, events })) } @@ -320,6 +366,9 @@ describe('runTick', () => { reverted: 0, submitted: 1, notSent: 0, + sendRefused: 0, + sendReverted: 0, + sendRejected: 0, unpriced: 0 }) expect(simulateCalls()).toBe(1) @@ -519,15 +568,25 @@ describe('runTick', () => { // Seeded at block 1 (suppressed until 3) so it does not suppress this tick at 100. A queue-wide // refusal says nothing about this position, so its history must survive un-extended: clearing // it is what let a failing position reset to attempt 1 and re-quote every other block. - const { counters, backoff, submitCalls } = await runWith({ + const { counters, backoff, submitCalls, streakCalls } = await runWith({ seedBackoffAt: 1n, submitOutcome: { sent: false, reason: 'refused' } }) expect(submitCalls()).toBe(1) - expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) + expect(counters).toMatchObject({ + ok: 1, + submitted: 0, + notSent: 1, + sendRefused: 1, + sendReverted: 0, + sendRejected: 0 + }) expect(backoff.shouldSkip(LABEL, 1n)).toBe(true) // Not re-armed: the next block may try again, which is the point of not blaming the position. expect(backoff.shouldSkip(LABEL, 101n)).toBe(false) + // Nothing was sent, so the chain said nothing about this plan: the streak must be neither + // extended nor reset. + expect(streakCalls).toEqual([]) expectCounterIdentities(counters) }) @@ -535,15 +594,103 @@ describe('runTick', () => { // The send itself failed, which is a fact about this position. Reaching submit at all means any // earlier entry had expired, so leaving it untouched would suppress nothing and the next block // would re-quote, re-simulate and re-send — the exact loop backoff exists to stop. - const { counters, backoff } = await runWith({ + const { counters, backoff, streakCalls } = await runWith({ seedBackoffAt: 1n, - submitOutcome: { sent: false, reason: 'send_failed' } + submitOutcome: { sent: false, reason: 'send_failed', executionRevert: false } + }) + expect(counters).toMatchObject({ + ok: 1, + submitted: 0, + notSent: 1, + sendRefused: 0, + sendReverted: 0, + sendRejected: 1 }) - expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) expect(backoff.shouldSkip(LABEL, 101n)).toBe(true) + // The chain never saw this plan, so it cannot extend a streak of the chain declining it. + expect(streakCalls).toEqual([{ kind: 'reset', label: LABEL }]) expectCounterIdentities(counters) }) + it('does not extend the suppression window when the CHAIN declined the send', async () => { + // The measured 2026-08-28 shape: `simulate.ok` at `latest`, then a min-out shortfall on the send. + // Post-maturity the LIF ramps on wall-clock, so that says nothing about the next block — backing + // off here is what sampled the ramp exponentially and produced 25-31s gaps between attempts. + const { counters, backoff, streakCalls } = await runWith({ + seedBackoffAt: 1n, + submitOutcome: { + sent: false, + reason: 'send_failed', + executionRevert: true, + selector: '0x08c379a0' + } + }) + expect(counters).toMatchObject({ + ok: 1, + submitted: 0, + notSent: 1, + sendRefused: 0, + sendReverted: 1, + sendRejected: 0 + }) + expect(backoff.shouldSkip(LABEL, 101n)).toBe(false) + // Reported, not suppressed: the streak is the backstop for a persistent estimator-only revert. + expect(streakCalls).toEqual([{ kind: 'record', label: LABEL, selector: '0x08c379a0' }]) + expectCounterIdentities(counters) + }) + + it('escalates a revert streak that has run past the threshold', async () => { + const { events } = await runWith({ + submitOutcome: { + sent: false, + reason: 'send_failed', + executionRevert: true, + selector: '0x08c379a0' + }, + revertStreak: { + count: 37, + durationMs: 16 * 60_000, + selector: '0x08c379a0', + selectorConstant: true, + escalate: 'crossed' + } + }) + const escalation = events.find(e => e.event === 'send.revert_streak') + expect(escalation?.level).toBe('warn') + // The count rides along with the duration so a count-based rule can be calibrated later, and the + // selector says whether the chain keeps refusing for the SAME reason. + expect(escalation?.fields).toMatchObject({ + borrower: BORROWER, + reverts: 37, + durationMs: 16 * 60_000, + selector: '0x08c379a0', + selectorConstant: true + }) + }) + + it('stays quiet while the streak is inside the threshold', async () => { + const { events } = await runWith({ + submitOutcome: { sent: false, reason: 'send_failed', executionRevert: true } + }) + expect(events.some(e => e.event === 'send.revert_streak')).toBe(false) + }) + + it('stays quiet on an already-escalated streak, so a stuck position warns once', async () => { + // This path has no throttle by design, so an `ongoing` streak would otherwise ship a warn every + // tick — twice on a tick where both siblings revert — for as long as the position stays stuck. + const { events } = await runWith({ + submitOutcome: { sent: false, reason: 'send_failed', executionRevert: true }, + revertStreak: { + count: 120, + durationMs: 60 * 60_000, + selector: '0x08c379a0', + selectorConstant: true, + escalate: 'ongoing' + } + }) + expect(events.some(e => e.event === 'send.revert_streak')).toBe(false) + }) + it('clears backoff and counts submitted only when the queue broadcast', async () => { const { counters, backoff } = await runWith({ seedBackoffAt: 1n, @@ -553,6 +700,11 @@ describe('runTick', () => { expect(backoff.shouldSkip(LABEL, 1n)).toBe(false) }) + it('ends the revert streak on a broadcast', async () => { + const { streakCalls } = await runWith({ submitOutcome: { sent: true } }) + expect(streakCalls).toEqual([{ kind: 'reset', label: LABEL }]) + }) + it('emits tick.end with complete: false when a submit aborts the tick', async () => { const { logger, events } = spyLogger() await expect( @@ -574,6 +726,7 @@ describe('runTick', () => { }, backoff: createBackoff({ baseBlocks: 2n, maxBlocks: 64n }), cooldown: createCooldownStore({ cooldownMs: 0 }), + revertStreaks: unreachableStreaks, inflightLabels: () => new Set(), usdValueOf: (_loanToken, loanUnits) => loanUnits, // A fully cold curve, so this case exercises the fail-open ordering it always did. @@ -587,6 +740,112 @@ describe('runTick', () => { ).rejects.toThrow('nonce claimed, no hash') const end = events.find(e => e.event === 'tick.end') expect(end?.fields).toMatchObject({ ok: 1, submitted: 0, notSent: 0, complete: false }) + // `expectCounterIdentities` cannot be used here: `ok === submitted + notSent` is intentionally + // short by one on this path. The `notSent` decomposition still holds, because the throw + // increments none of its four terms. + // Read per key rather than casting the whole bag: a missing counter must fail this assertion + // (`Number(undefined)` is NaN), not be typed into existence. + const counted = (key: string) => Number(end?.fields?.[key]) + expect(counted('notSent')).toBe( + counted('sendRefused') + counted('sendReverted') + counted('sendRejected') + ) + }) + + // `pendingBackoff` is keyed by POSITION and shared by every sibling candidate, so not ARMING it on + // an execution revert is necessary but not sufficient: a sibling arms it on either side of the + // reverted send. Unlike a broadcast, an execution revert does not enter `submittedLabels`, so the + // next-ranked candidate still runs — hence both orderings below, which is what a delete rather than + // a latch gets wrong on the common multi-collateral position. + describe('sibling precedence', () => { + const declined: SubmitOutcome = { + sent: false, + reason: 'send_failed', + executionRevert: true, + selector: '0x08c379a0' + } + + it('a quote failure then an execution revert leaves the position unsuppressed', async () => { + const { counters, backoff } = await runWith({ + out: twoSlots(), + quoteOutcomes: [ + { kind: 'failed', reason: 'timeout' }, + { kind: 'swap', plan: SWAP_PLAN } + ], + submitOutcome: declined + }) + expect(counters).toMatchObject({ candidates: 2, quoteFailed: 1, ok: 1, sendReverted: 1 }) + expect(backoff.shouldSkip(LABEL, 101n)).toBe(false) + expectCounterIdentities(counters) + }) + + it('a no-route quote then an execution revert leaves the position unsuppressed', async () => { + const { counters, backoff } = await runWith({ + out: twoSlots(), + quoteOutcomes: [ + { kind: 'failed', reason: 'no_route' }, + { kind: 'swap', plan: SWAP_PLAN } + ], + submitOutcome: declined + }) + expect(counters).toMatchObject({ candidates: 2, quoteFailed: 1, ok: 1, sendReverted: 1 }) + expect(backoff.shouldSkip(LABEL, 101n)).toBe(false) + expectCounterIdentities(counters) + }) + + it('a simulation revert then an execution revert leaves the position unsuppressed', async () => { + const { counters, backoff } = await runWith({ + out: twoSlots(), + simulateResults: [{ status: 'revert', reason: 'stale quote' }, { status: 'ok' }], + submitOutcome: declined + }) + expect(counters).toMatchObject({ candidates: 2, reverted: 1, ok: 1, sendReverted: 1 }) + expect(backoff.shouldSkip(LABEL, 101n)).toBe(false) + expectCounterIdentities(counters) + }) + + it('an execution revert then a quote failure leaves the position unsuppressed', async () => { + const { counters, backoff } = await runWith({ + out: twoSlots(), + quoteOutcomes: [ + { kind: 'swap', plan: SWAP_PLAN }, + { kind: 'failed', reason: 'no_route' } + ], + submitOutcome: declined + }) + expect(counters).toMatchObject({ candidates: 2, quoteFailed: 1, ok: 1, sendReverted: 1 }) + expect(backoff.shouldSkip(LABEL, 101n)).toBe(false) + expectCounterIdentities(counters) + }) + + it('an execution revert then a simulation revert leaves the position unsuppressed', async () => { + const { counters, backoff } = await runWith({ + out: twoSlots(), + simulateResults: [{ status: 'ok' }, { status: 'revert', reason: 'stale quote' }], + submitOutcome: declined + }) + expect(counters).toMatchObject({ candidates: 2, reverted: 1, ok: 1, sendReverted: 1 }) + expect(backoff.shouldSkip(LABEL, 101n)).toBe(false) + expectCounterIdentities(counters) + }) + + it('leaves an opted-in cooldown armed, unlike backoff', async () => { + // The cooldown is a flat, default-off window an operator opts into to throttle a class of + // positions; it never samples the LIF ramp geometrically the way backoff does, so lifting it is + // an operator's call rather than an inference from one sibling's outcome. + const cooldown = createCooldownStore({ cooldownMs: 60_000 }) + const { counters, backoff } = await runWith({ + cooldown, + out: twoSlots(), + quoteOutcomes: [ + { kind: 'failed', reason: 'no_route' }, + { kind: 'swap', plan: SWAP_PLAN } + ], + submitOutcome: declined + }) + expect(counters).toMatchObject({ quoteFailed: 1, sendReverted: 1 }) + expect(backoff.shouldSkip(LABEL, 101n)).toBe(false) + expect(cooldown.shouldSkip(LABEL)).toBe(true) + }) }) }) @@ -880,14 +1139,6 @@ describe('runTick', () => { }) describe('multi-collateral candidate fall-through', () => { - // A two-slot position: `collateralParams[1]` (COLLATERAL, 86% lltv) needs a swap and outranks - // `collateralParams[0]` (the loan token itself, 98% lltv) on surplus, so it is tried first. - const twoSlots = () => - lensOut({ - activatedBitmap: 0b11n, - collaterals: [slot({ index: 1 }), slot({ index: 0, maxLif: LOAN_MAX_LIF })] - }) - it('counts one planned POSITION but two CANDIDATES', async () => { // The two `tick.end` identities count different things; this is the case that separates them. const { counters } = await runWith({ out: twoSlots() }) diff --git a/bots/vault-v1-reallocation/src/index.ts b/bots/vault-v1-reallocation/src/index.ts index ab84f370..a0e32b61 100644 --- a/bots/vault-v1-reallocation/src/index.ts +++ b/bots/vault-v1-reallocation/src/index.ts @@ -21,9 +21,9 @@ import { getBlockNumber, readContract } from 'viem/actions' import { loadConfig } from './config' import { createIntervalGate } from './interval-gate' +import { revertReason } from './revert.utils' import { runTick } from './runner/tick' import { createStrategy } from './strategies' -import { revertReason } from './tx-error' import { checkVaults } from './vault-checks' import { fetchVaultData } from './vault-data' diff --git a/bots/vault-v1-reallocation/src/tx-error.ts b/bots/vault-v1-reallocation/src/revert.utils.ts similarity index 100% rename from bots/vault-v1-reallocation/src/tx-error.ts rename to bots/vault-v1-reallocation/src/revert.utils.ts diff --git a/bots/vault-v1-reallocation/test/tx-error.test.ts b/bots/vault-v1-reallocation/test/revert.utils.test.ts similarity index 87% rename from bots/vault-v1-reallocation/test/tx-error.test.ts rename to bots/vault-v1-reallocation/test/revert.utils.test.ts index 5d621797..81a80881 100644 --- a/bots/vault-v1-reallocation/test/tx-error.test.ts +++ b/bots/vault-v1-reallocation/test/revert.utils.test.ts @@ -2,10 +2,10 @@ import { metaMorphoAbi } from '@morpho-org/blue-sdk-viem' import { BaseError, encodeErrorResult } from 'viem' import { describe, expect, it } from 'vitest' -import { revertReason } from '../src/tx-error' +import { revertReason } from '../src/revert.utils' // A viem-style error chain whose cause carries an ABI-encoded revert payload, the shape -// `revertReason` walks for (mirrors bot-kit's own tx-error tests). +// `revertReason` walks for (mirrors bot-kit's own revert.utils tests). const revertError = (data: `0x${string}`): BaseError => new BaseError('execution reverted', { cause: Object.assign(new Error('execution reverted'), { data }) diff --git a/bots/vault-v2-reallocation/src/index.ts b/bots/vault-v2-reallocation/src/index.ts index fbd8627a..98d3e7b0 100644 --- a/bots/vault-v2-reallocation/src/index.ts +++ b/bots/vault-v2-reallocation/src/index.ts @@ -21,9 +21,9 @@ import { getBlockNumber } from 'viem/actions' import { loadConfig } from './config' import { encodeReallocation } from './encode' import { createIntervalGate } from './interval-gate' +import { revertReason } from './revert.utils' import { runTick } from './runner/tick' import { createStrategy } from './strategies' -import { revertReason } from './tx-error' import { checkVaults } from './vault-checks' import { fetchVaultV2Data } from './vault-data' diff --git a/bots/vault-v2-reallocation/src/tx-error.ts b/bots/vault-v2-reallocation/src/revert.utils.ts similarity index 100% rename from bots/vault-v2-reallocation/src/tx-error.ts rename to bots/vault-v2-reallocation/src/revert.utils.ts diff --git a/bots/vault-v2-reallocation/test/tx-error.test.ts b/bots/vault-v2-reallocation/test/revert.utils.test.ts similarity index 87% rename from bots/vault-v2-reallocation/test/tx-error.test.ts rename to bots/vault-v2-reallocation/test/revert.utils.test.ts index 7964b4b1..7ebe2bb8 100644 --- a/bots/vault-v2-reallocation/test/tx-error.test.ts +++ b/bots/vault-v2-reallocation/test/revert.utils.test.ts @@ -2,10 +2,10 @@ import { vaultV2Abi } from '@morpho-org/blue-sdk-viem' import { BaseError, encodeErrorResult } from 'viem' import { describe, expect, it } from 'vitest' -import { revertReason } from '../src/tx-error' +import { revertReason } from '../src/revert.utils' // A viem-style error chain whose cause carries an ABI-encoded revert payload, the shape -// `revertReason` walks for (mirrors bot-kit's own tx-error tests). +// `revertReason` walks for (mirrors bot-kit's own revert.utils tests). const revertError = (data: `0x${string}`): BaseError => new BaseError('execution reverted', { cause: Object.assign(new Error('execution reverted'), { data }) diff --git a/packages/bot-kit/src/index.ts b/packages/bot-kit/src/index.ts index 5875a74e..36f1f017 100644 --- a/packages/bot-kit/src/index.ts +++ b/packages/bot-kit/src/index.ts @@ -8,9 +8,10 @@ export * from './queue/backoff' export * from './queue/cooldown' export * from './queue/fee-policy' export * from './queue/pending-queue' +export * from './revert.utils' export * from './runner/runner' export * from './runner/watcher' export * from './shipping-config' export * from './signer' export * from './simulate' -export * from './tx-error' +export * from './tx-send.error' diff --git a/packages/bot-kit/src/queue/pending-queue.ts b/packages/bot-kit/src/queue/pending-queue.ts index bea666bd..6e98d029 100644 --- a/packages/bot-kit/src/queue/pending-queue.ts +++ b/packages/bot-kit/src/queue/pending-queue.ts @@ -5,7 +5,12 @@ import { tryCatch } from '@repo/utils' import type { Logger } from '../logger' -import { isExecutionRevert, revertReason as defaultRevertReason, TxSendError } from '../tx-error' +import { + isExecutionRevert, + revertReason as defaultRevertReason, + revertSelector +} from '../revert.utils' +import { TxSendError } from '../tx-send.error' import { bumpFees } from './fee-policy' /** Default blocks a pending tx may sit unconfirmed before the queue bumps its fee and replaces it. */ @@ -61,9 +66,20 @@ export type SubmitArgs = { * (`tx.send_aborted`, `nonce.sync_failed`, `queue.nonce_hole`) — a queue-wide condition that would * have refused any position, so a caller must NOT hold it against this one. `send_failed` means the * node rejected this position's own transaction (`tx.submit_failed`), which is a fact about the - * position and should re-arm whatever per-position backoff the caller keeps. + * position — though whether that fact re-arms a caller's per-position backoff turns on the + * `executionRevert` split below. + * + * `send_failed` carries `executionRevert`, which splits that fact in two. `true` means the chain + * declined this plan right now — a caller may treat it as economic. `false` means the send machinery + * failed (nonce, funds, RPC) and nothing was learned about the plan itself. `selector` is the revert + * payload's 4-byte selector when it carried one: a caller watching consecutive declines needs to know + * whether the chain keeps refusing for the SAME reason, which the decoded message cannot be relied on + * to say. */ -export type SubmitOutcome = { sent: true } | { sent: false; reason: 'refused' | 'send_failed' } +export type SubmitOutcome = + | { sent: true } + | { sent: false; reason: 'refused' } + | { sent: false; reason: 'send_failed'; executionRevert: boolean; selector?: Hex } /** One tracked tx — the queue's full per-nonce record. */ type Pending = { @@ -86,7 +102,9 @@ export type PendingQueue = { * * On failure the outcome distinguishes {@link SubmitOutcome}'s two reasons, which callers must not * collapse: `refused` is queue-wide and says nothing about this position, while `send_failed` is - * this position's own send being rejected. + * this position's own send being rejected. Nor may a caller collapse `send_failed`'s + * `executionRevert` split — an execution revert is the chain's verdict on the plan, a false one is + * the send machinery failing, and only the latter is evidence about the position. * * Concurrent calls are serialized end to end (latch checks → `syncNonce` → `send` → tracking), so a * pass that submits for several positions at once cannot hand two of them the same nonce and cannot @@ -264,9 +282,13 @@ export function createPendingQueue({ }) ) if (sent.error) { + const executionRevert = isExecutionRevert(sent.error) + const selector = revertSelector(sent.error) logger.warn('tx.submit_failed', { label: args.label, reason: revertReason(sent.error), + executionRevert, + ...(selector ? { selector } : {}), ...(sent.error instanceof TxSendError && sent.error.nonce !== undefined ? { nonce: sent.error.nonce } : {}) @@ -278,7 +300,12 @@ export function createPendingQueue({ sendAborted = true throw sent.error } - return { sent: false, reason: 'send_failed' } + return { + sent: false, + reason: 'send_failed', + executionRevert, + ...(selector ? { selector } : {}) + } } const { nonce, txHash } = sent.data pending.set(nonce, { diff --git a/packages/bot-kit/src/tx-error.ts b/packages/bot-kit/src/revert.utils.ts similarity index 58% rename from packages/bot-kit/src/tx-error.ts rename to packages/bot-kit/src/revert.utils.ts index 46249af2..50f3bf02 100644 --- a/packages/bot-kit/src/tx-error.ts +++ b/packages/bot-kit/src/revert.utils.ts @@ -1,49 +1,50 @@ import type { Abi, Hex } from 'viem' -import { BaseError, decodeErrorResult, ExecutionRevertedError } from 'viem' +import { BaseError, decodeErrorResult, ExecutionRevertedError, size, slice } from 'viem' -/** - * Raised by the signer when an initial broadcast fails after the bot has already claimed a nonce. - * The tx hash is unknown, so the queue cannot track a pending hash; callers must treat this as a - * tick-level failure and retry after the signer rolls its local nonce cursor back. - */ -export class TxSendError extends Error { - readonly nonce: number | undefined - readonly originalError: unknown +import { TxSendError } from './tx-send.error' - constructor(error: unknown, nonce?: number) { - super(error instanceof Error ? error.message : String(error)) - this.name = 'TxSendError' - this.nonce = nonce - this.originalError = error - } -} +const SELECTOR_BYTES = 4 -function unwrapSendError(error: unknown): unknown { - return error instanceof TxSendError ? error.originalError : error -} +const unwrapSendError = (error: unknown): unknown => + error instanceof TxSendError ? error.originalError : error /** * True if `error` is an on-chain execution revert (the tx cannot succeed) rather than a transient * RPC/network error (timeout, HTTP, nonce). The pending queue uses this to decide whether a stuck - * tx should be dropped (a revert won't fix itself, so bumping is futile) or retried (transient). + * tx should be dropped (a revert won't fix itself, so bumping is futile) or retried (transient). It + * is also what the queue reports as `SubmitOutcome.executionRevert`, so a bot whose incentive ramps + * on wall-clock decides on this predicate alone whether a failed send suppresses the position. */ -export function isExecutionRevert(error: unknown): boolean { - error = unwrapSendError(error) - if (!(error instanceof BaseError)) return false - if (error.walk(e => e instanceof ExecutionRevertedError) !== null) return true +export const isExecutionRevert = (error: unknown): boolean => { + const unwrapped = unwrapSendError(error) + if (!(unwrapped instanceof BaseError)) return false + if (unwrapped.walk(e => e instanceof ExecutionRevertedError) !== null) return true // Some transports surface the canonical message without viem's typed subclass. - return /execution reverted/i.test(error.shortMessage) + return /execution reverted/i.test(unwrapped.shortMessage) } /** The ABI-encoded revert payload a node returned, if any (searched across the viem error chain). */ -function revertData(error: BaseError): Hex | undefined { +const revertData = (error: BaseError): Hex | undefined => { const withData = error.walk(e => typeof (e as { data?: unknown }).data === 'string') as { data?: Hex } | null return withData?.data?.startsWith('0x') ? withData.data : undefined } +/** + * The revert payload's 4-byte selector, or `undefined` when the error carries no payload (or one too + * short to hold a selector). Attributes a revert that {@link revertReason} could not decode, since + * the selector identifies the reverting contract's error even when no ABI in reach defines it. + */ +export const revertSelector = (error: unknown): Hex | undefined => { + const unwrapped = unwrapSendError(error) + if (!(unwrapped instanceof BaseError)) return undefined + const data = revertData(unwrapped) + if (!data || size(data) < SELECTOR_BYTES) return undefined + return slice(data, 0, SELECTOR_BYTES) +} + /** * Decodes an ABI-encoded revert payload to a log-safe string. May throw on an unknown selector — * {@link revertReason} falls through to viem's short message when it does. @@ -70,12 +71,12 @@ const decodeStandardRevert: RevertDecoder = data => { * viem's `decodeErrorResult` also handles the standard `Error`/`Panic` selectors with a custom ABI, * so the returned decoder covers standard reverts too. */ -export function abiRevertDecoder(abi: Abi): RevertDecoder { - return data => { +export const abiRevertDecoder = + (abi: Abi): RevertDecoder => + data => { const { errorName, args } = decodeErrorResult({ abi, data }) return args && args.length > 0 ? `${errorName}(${args.join(', ')})` : errorName } -} /** * A concise, log-safe failure reason: the decoded revert (via `decode`, defaulting to the standard @@ -83,10 +84,15 @@ export function abiRevertDecoder(abi: Abi): RevertDecoder { * the request/calldata dump that bloats `error.message` (and gets truncated by log shippers). * Protocols with custom ABI errors pass `abiRevertDecoder(theirAbi)`. */ -export function revertReason(error: unknown, decode: RevertDecoder = decodeStandardRevert): string { - error = unwrapSendError(error) - if (!(error instanceof BaseError)) return error instanceof Error ? error.message : String(error) - const data = revertData(error) +export const revertReason = ( + error: unknown, + decode: RevertDecoder = decodeStandardRevert +): string => { + const unwrapped = unwrapSendError(error) + if (!(unwrapped instanceof BaseError)) { + return unwrapped instanceof Error ? unwrapped.message : String(unwrapped) + } + const data = revertData(unwrapped) if (data) { try { return decode(data) @@ -94,5 +100,5 @@ export function revertReason(error: unknown, decode: RevertDecoder = decodeStand // Not an error shape the decoder knows — fall through to viem's short message. } } - return error.shortMessage + return unwrapped.shortMessage } diff --git a/packages/bot-kit/src/runner/runner.ts b/packages/bot-kit/src/runner/runner.ts index e3fddbb9..4c64a50f 100644 --- a/packages/bot-kit/src/runner/runner.ts +++ b/packages/bot-kit/src/runner/runner.ts @@ -2,7 +2,7 @@ import { tryCatch } from '@repo/utils' import type { Logger } from '../logger' -import { revertReason as defaultRevertReason } from '../tx-error' +import { revertReason as defaultRevertReason } from '../revert.utils' import { BLOCK_POLL_MS, createBlockWatcher } from './watcher' /** The long-running lifecycle handle {@link createRunner} returns. */ diff --git a/packages/bot-kit/src/signer.ts b/packages/bot-kit/src/signer.ts index 9b17dea6..0aa5c647 100644 --- a/packages/bot-kit/src/signer.ts +++ b/packages/bot-kit/src/signer.ts @@ -24,7 +24,7 @@ import type { import { evaluatePolicy, PolicyViolationError } from './policy' import { createHttpTransport } from './transport' -import { TxSendError } from './tx-error' +import { TxSendError } from './tx-send.error' /** The signed-send primitives {@link createSigner} returns and `createPendingQueue` injects. */ export type Signer = { diff --git a/packages/bot-kit/src/tx-send.error.ts b/packages/bot-kit/src/tx-send.error.ts new file mode 100644 index 00000000..491d03d6 --- /dev/null +++ b/packages/bot-kit/src/tx-send.error.ts @@ -0,0 +1,16 @@ +/** + * Raised by the signer when an initial broadcast fails after the bot has already claimed a nonce. + * The tx hash is unknown, so the queue cannot track a pending hash; callers must treat this as a + * tick-level failure and retry after the signer rolls its local nonce cursor back. + */ +export class TxSendError extends Error { + readonly nonce: number | undefined + readonly originalError: unknown + + constructor(error: unknown, nonce?: number) { + super(error instanceof Error ? error.message : String(error)) + this.name = 'TxSendError' + this.nonce = nonce + this.originalError = error + } +} diff --git a/packages/bot-kit/test/queue/pending-queue.test.ts b/packages/bot-kit/test/queue/pending-queue.test.ts index 7dbdd4d1..c1524eb5 100644 --- a/packages/bot-kit/test/queue/pending-queue.test.ts +++ b/packages/bot-kit/test/queue/pending-queue.test.ts @@ -1,6 +1,6 @@ import type { Address, Hex } from 'viem' -import { ExecutionRevertedError } from 'viem' +import { encodeErrorResult, ExecutionRevertedError } from 'viem' import { describe, expect, it } from 'vitest' import type { Logger, LogLevel } from '../../src/logger' @@ -15,7 +15,7 @@ import type { } from '../../src/queue/pending-queue' import { createPendingQueue } from '../../src/queue/pending-queue' -import { TxSendError } from '../../src/tx-error' +import { TxSendError } from '../../src/tx-send.error' // The cooldown the opted-in cases run with (mirrors midnight's SETTLED_COOLDOWN_BLOCKS tuning). const SETTLED_COOLDOWN_BLOCKS = 20n @@ -639,7 +639,52 @@ describe('submit outcome', () => { // The node rejected THIS position's transaction — a fact about the position, so a caller must be // able to re-arm its backoff. Collapsing this with a queue refusal is what let a failing send // re-quote and re-send every block. - expect(await submitOne(queue)).toEqual({ sent: false, reason: 'send_failed' }) + expect(await submitOne(queue)).toEqual({ + sent: false, + reason: 'send_failed', + executionRevert: false + }) + }) + + it('reports executionRevert on a send the chain reverted, and logs its selector', async () => { + const data = encodeErrorResult({ + abi: [{ type: 'error', name: 'Error', inputs: [{ type: 'string' }] }] as const, + errorName: 'Error', + args: ['return too low'] + }) + const send: SendTx = async () => { + throw Object.assign(new ExecutionRevertedError({}), { data }) + } + const { logger, events } = captureLogger() + const { queue } = setup({ send, logger }) + // The selector rides the outcome, not just the log line: the caller watching consecutive declines + // needs to know whether the chain keeps refusing for the same reason. + expect(await submitOne(queue)).toEqual({ + sent: false, + reason: 'send_failed', + executionRevert: true, + selector: data.slice(0, 10) + }) + const failed = events.find(e => e.event === 'tx.submit_failed') + expect(failed?.fields?.reason).toBe('return too low') + expect(failed?.fields?.executionRevert).toBe(true) + expect(failed?.fields?.selector).toBe(data.slice(0, 10)) + }) + + it('omits the selector when a transport failure carries no revert payload', async () => { + const send: SendTx = async () => { + throw new Error('nonce too low') + } + const { logger, events } = captureLogger() + const { queue } = setup({ send, logger }) + expect(await submitOne(queue)).toEqual({ + sent: false, + reason: 'send_failed', + executionRevert: false + }) + const failed = events.find(e => e.event === 'tx.submit_failed') + expect(failed?.fields?.executionRevert).toBe(false) + expect(failed?.fields).not.toHaveProperty('selector') }) it('reports refused when the empty-queue nonce re-sync throws', async () => { diff --git a/packages/bot-kit/test/tx-error.test.ts b/packages/bot-kit/test/revert.utils.test.ts similarity index 76% rename from packages/bot-kit/test/tx-error.test.ts rename to packages/bot-kit/test/revert.utils.test.ts index 7ad643cc..24a760c5 100644 --- a/packages/bot-kit/test/tx-error.test.ts +++ b/packages/bot-kit/test/revert.utils.test.ts @@ -1,7 +1,13 @@ import { BaseError, encodeErrorResult, ExecutionRevertedError } from 'viem' import { describe, expect, it } from 'vitest' -import { abiRevertDecoder, isExecutionRevert, revertReason, TxSendError } from '../src/tx-error' +import { + abiRevertDecoder, + isExecutionRevert, + revertReason, + revertSelector +} from '../src/revert.utils' +import { TxSendError } from '../src/tx-send.error' const SOLIDITY_ERRORS = [ { type: 'error', name: 'Error', inputs: [{ type: 'string' }] }, @@ -106,3 +112,26 @@ describe('isExecutionRevert', () => { expect(isExecutionRevert(new BaseError('http timeout'))).toBe(false) }) }) + +describe('revertSelector', () => { + it('returns the 4-byte selector of a revert payload', () => { + const data = encodeErrorResult({ abi: CUSTOM_ABI, errorName: 'NotBorrower', args: [BORROWER] }) + expect(revertSelector(revertError(data))).toBe(data.slice(0, 10)) + }) + + it('unwraps a TxSendError so a hashless send is classified like its inner error', () => { + const data = encodeErrorResult({ abi: CUSTOM_ABI, errorName: 'Halted', args: [] }) + expect(revertSelector(new TxSendError(revertError(data), 7))).toBe(data.slice(0, 10)) + }) + + it('returns undefined for errors carrying no revert payload', () => { + expect(revertSelector(new BaseError('http timeout'))).toBeUndefined() + expect(revertSelector(new Error('connection reset'))).toBeUndefined() + expect(revertSelector('string failure')).toBeUndefined() + }) + + it('returns undefined for a payload too short to hold a selector', () => { + expect(revertSelector(revertError('0x'))).toBeUndefined() + expect(revertSelector(revertError('0xdead'))).toBeUndefined() + }) +}) diff --git a/packages/bot-kit/test/signer.test.ts b/packages/bot-kit/test/signer.test.ts index a0c77cd3..627def19 100644 --- a/packages/bot-kit/test/signer.test.ts +++ b/packages/bot-kit/test/signer.test.ts @@ -8,7 +8,7 @@ import type { Policy } from '../src/policy' import { EXECUTOR_SELECTOR, PolicyViolationError } from '../src/policy' import { createSigner } from '../src/signer' -import { TxSendError } from '../src/tx-error' +import { TxSendError } from '../src/tx-send.error' const EXECUTOR = `0x${'11'.repeat(20)}` as const const POLICY: Policy = {