From 44003519140980b366582c8469684ce373d53f12 Mon Sep 17 00:00:00 2001 From: Hayden Shively Date: Mon, 31 Aug 2026 12:00:38 -0500 Subject: [PATCH 1/2] fix(bots): normalize the position join key across log events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every position-scoped log event in both liquidators now carries the position in one field, `id`, valued `lensKey(marketId, borrower)` — the composite the helper already lowercases on both halves. A maturity's events therefore group into one row per position with no normalization in the query. Before this, the same position was identified three different ways: `marketId` + `borrower` (checksummed) on the plan and simulate events, a single lowercased `id` on the select and quote events, and that same string under the name `label` on `tx.*`. Grouping by the raw composite returned 26 rows for 13 positions on the 2026-08-28 maturity, which reads as if the bot planned positions it never quoted. `marketId` and `borrower` stay on `plan.built` as human-readable extras. Since a tick is multi-collateral, `id` identifies a POSITION and the per-candidate key is `(id, collateralIndex, postMaturityMode)`. That pair now threads through `QuoteRequest.candidate` so the `@repo/swaps` quote events can separate two candidates of one position; it inherits `id`'s correlation-only contract and nothing in the package branches on it. In the pending queue only the EMITTED field is renamed. `SubmitArgs.label` and the in-flight map keys keep their name because they are behavioral: `inflightLabels().has(label)` is tested every tick, so touching the value would let a second nonce-consuming send go out for a position already in flight. That also changes the `tx.*` log schema of the reallocation bots and midnight-crossed-books, which pass a vault address or a market id as the same key. Closes BOTS-90. Co-Authored-By: Claude Opus 5 (1M context) --- bots/blue-liquidation/README.md | 12 ++++ bots/blue-liquidation/src/quotes.ts | 5 +- bots/blue-liquidation/src/runner/tick.ts | 9 ++- .../blue-liquidation/test/runner/tick.test.ts | 39 ++++++++++++- bots/midnight-liquidation/README.md | 25 ++++++++- bots/midnight-liquidation/src/quotes.ts | 15 ++++- bots/midnight-liquidation/src/runner/tick.ts | 31 ++++++++-- bots/midnight-liquidation/test/quotes.test.ts | 20 +++++++ .../test/runner/tick.test.ts | 56 ++++++++++++++++++- ...TIB-2026-05-28-midnight-liquidation-bot.md | 7 +++ ...-midnight-send-shortfall-classification.md | 18 ++++-- packages/bot-kit/src/queue/pending-queue.ts | 36 +++++++----- .../bot-kit/test/queue/pending-queue.test.ts | 8 +-- packages/swaps/src/quoting.ts | 53 +++++++++++++----- .../src/helpers/deployless-batch-lens.ts | 5 ++ 15 files changed, 280 insertions(+), 59 deletions(-) diff --git a/bots/blue-liquidation/README.md b/bots/blue-liquidation/README.md index b895a4e7..f5510d4c 100644 --- a/bots/blue-liquidation/README.md +++ b/bots/blue-liquidation/README.md @@ -260,6 +260,18 @@ distinct nonces, EIP-1559 ≥12.5% fee bump on stuck nonces, and a hard `MAX_FEE drops rather than chases a gas spike. State is not persisted — chain truth wins; a restart re-derives the nonce from `getTransactionCount('pending')`. +### Log correlation + +Every position-scoped event — `plan.*`, `cooldown.*`, `config.*`, `quote.*`, `unwrap.*`, `select.*`, +`simulate.*`, `tx.*`, `queue.*`, `nonce.*` — carries the position in one field, **`id`**, whose value +is `lensKey(marketId, borrower)`: the two halves joined by `:` with both lowercased. So a window's +events group into one row per position with **no normalization in the query** (`GROUP BY id`). `tx.*` +used to name the same string `label`; it does not any more. `plan.built` also keeps `marketId` and +`borrower` as human-readable extras — for reading a single line, not for grouping. + +A Blue market has exactly one collateral, so one position is one candidate: unlike +`bots/midnight-liquidation`, `id` alone identifies a row and no candidate discriminator is emitted. + ## Testing - `pnpm test` — unit tests for the math, LIF, seize-exact planner (incl. the underflow-safety sweep), diff --git a/bots/blue-liquidation/src/quotes.ts b/bots/blue-liquidation/src/quotes.ts index 31619988..386d7533 100644 --- a/bots/blue-liquidation/src/quotes.ts +++ b/bots/blue-liquidation/src/quotes.ts @@ -47,7 +47,10 @@ export function composeQuoting(deps: { // The operator opt-out applies to the RAW collateral — blue has no per-collateral config file // anymore, so this is its escape hatch from the auto-unwrap path too. if (excludeCollaterals.some(token => isAddressEqual(token, out.params.collateralToken))) { - logger.info('quote.excluded_collateral', { collateral: out.params.collateralToken }) + logger.info('quote.excluded_collateral', { + id: label, + collateral: out.params.collateralToken + }) // `firmCalls: 0` explicitly: an absent count reads as UNKNOWN, and this path provably spent // nothing (see {@link QuoteOutcome.firmCalls}). return { kind: 'no_config', firmCalls: 0 } diff --git a/bots/blue-liquidation/src/runner/tick.ts b/bots/blue-liquidation/src/runner/tick.ts index 9fb332ff..c6a0fe80 100644 --- a/bots/blue-liquidation/src/runner/tick.ts +++ b/bots/blue-liquidation/src/runner/tick.ts @@ -146,6 +146,9 @@ export async function runTick(deps: { if (!liquidationPlan) continue counters.planned += 1 logger.info('plan.built', { + id: label, + // Human-readable extras only: the pair `id` is built from, kept for an operator reading one + // line. Grouping keys on `id`. marketId: id, borrower: pair.borrower, seizedAssets: liquidationPlan.seizedAssets @@ -156,7 +159,7 @@ export async function runTick(deps: { // disabled (POSITION_LIQUIDATION_COOLDOWN_MS=0). if (cooldown.shouldSkip(label)) { counters.cooledDown += 1 - logger.info('cooldown.skip', { marketId: id, borrower: pair.borrower }) + logger.info('cooldown.skip', { id: label, marketId: id, borrower: pair.borrower }) continue } // Suppress positions that keep failing to quote/simulate — bounds API + RPC usage under a @@ -169,7 +172,7 @@ export async function runTick(deps: { if (outcome.kind === 'no_config') { counters.noSwapPath += 1 cooldown.mark(label) - logger.info('config.no_swap_path', { marketId: id, borrower: pair.borrower }) + logger.info('config.no_swap_path', { id: label, marketId: id, borrower: pair.borrower }) continue } if (outcome.kind === 'failed') { @@ -186,7 +189,7 @@ export async function runTick(deps: { plan: liquidationPlan, swapPlan }) - const fields = { marketId: id, borrower: pair.borrower } + const fields = { id: label, marketId: id, borrower: pair.borrower } switch (result.status) { case 'ok': counters.ok += 1 diff --git a/bots/blue-liquidation/test/runner/tick.test.ts b/bots/blue-liquidation/test/runner/tick.test.ts index bb14c184..a0ed44fb 100644 --- a/bots/blue-liquidation/test/runner/tick.test.ts +++ b/bots/blue-liquidation/test/runner/tick.test.ts @@ -3,7 +3,7 @@ import type { CooldownStore } from '@repo/bot-kit' import type { QuoteOutcome, SwapPlan } from '@repo/swaps' import type { Address } from 'viem' -import { createBackoff, createCooldownStore } from '@repo/bot-kit' +import { createBackoff, createCooldownStore, createPendingQueue } from '@repo/bot-kit' import { lensKey } from '@repo/utils' import { getAddress } from 'viem' import { describe, expect, it } from 'vitest' @@ -103,8 +103,12 @@ function runWith(opts: { cooldown?: CooldownStore /** Models the queue's outcome; the two no-broadcast reasons are NOT interchangeable. */ submitOutcome?: SubmitOutcome + /** Shared spy, so a caller can observe the tick's and the queue's events in ONE stream. */ + spy?: ReturnType + /** Replaces the stub `submit` — used to broadcast through a real pending queue. */ + submitWith?: (args: { label: string; blockNumber: bigint }) => Promise }) { - const { logger, events } = spyLogger() + const { logger, events } = opts.spy ?? spyLogger() let simulateCalls = 0 let submitCalls = 0 let quoteCalls = 0 @@ -131,8 +135,9 @@ function runWith(opts: { simulateCalls += 1 return opts.simulateResult ?? { status: 'ok' } }, - submit: async () => { + submit: async args => { submitCalls += 1 + if (opts.submitWith) return opts.submitWith(args) return opts.submitOutcome ?? { sent: true } }, backoff, @@ -375,4 +380,32 @@ describe('runTick', () => { expect(backoff.shouldSkip(LABEL, 1n)).toBe(false) }) }) + it('emits one id that joins plan.built to the queue tx.sent', async () => { + // BOTS-90's acceptance criterion as a test: grouping a window's events by `id` must not split one + // position. Broadcast through the REAL queue, since the split was between the tick's field name + // and the queue's — a stubbed submit cannot see it. + const spy = spyLogger() + const queue = createPendingQueue({ + send: async () => ({ nonce: 7, txHash: `0x${'1'.repeat(64)}` }), + getReceipt: async () => null, + getBaseFee: async () => 1n, + maxFeeWei: 10n ** 18n, + logger: spy.logger + }) + await runWith({ + spy, + submitWith: ({ label, blockNumber }) => + queue.submit({ + request: { to: ROUTER, data: '0x' }, + label, + maxFeePerGas: 1000n, + maxPriorityFeePerGas: 1000n, + blockNumber + }) + }) + const planBuilt = spy.events.find(e => e.event === 'plan.built') + const txSent = spy.events.find(e => e.event === 'tx.sent') + expect(planBuilt?.fields?.id).toBe(LABEL) + expect(txSent?.fields?.id).toBe(planBuilt?.fields?.id) + }) }) diff --git a/bots/midnight-liquidation/README.md b/bots/midnight-liquidation/README.md index 60fd3f47..625c949a 100644 --- a/bots/midnight-liquidation/README.md +++ b/bots/midnight-liquidation/README.md @@ -540,10 +540,10 @@ does not broadcast. On simulation success, `@repo/bot-kit`'s shared pending queue ([packages/bot-kit/src/queue/pending-queue.ts](../../packages/bot-kit/src/queue/pending-queue.ts)) -sends the transaction through the signer client and tracks it by nonce and `(marketId, borrower)` -label. +sends the transaction through the signer client and tracks it by nonce and by the position's +`(marketId, borrower)` key. -While a label is pending, later ticks skip that position. On each block the queue checks receipts, +While that position is in flight, later ticks skip it. On each block the queue checks receipts, logs confirmed or reverted transactions, and fee-bumps stuck transactions until either they confirm, hit the fee ceiling, or exhaust bump attempts. @@ -588,6 +588,25 @@ signer nonce cursor starts from the pending chain nonce. If the initial raw broa nonce is claimed but before a hash is returned, the signer rolls the cursor back and the queue aborts that tick instead of counting a hashless transaction as submitted. +### Log Correlation + +Every position-scoped event — `plan.*`, `preselect.*`, `route.*`, `cooldown.*`, `config.*`, `quote.*`, +`unwrap.*`, `select.*`, `simulate.*`, `send.*`, `tx.*`, `queue.*`, `nonce.*` — carries the position in +one field, **`id`**, whose value is `lensKey(marketId, borrower)`: the two halves joined by `:` with +both lowercased. So a maturity's events group into one row per position with **no normalization in the +query** (`GROUP BY id`). `tx.*` used to name the same string `label`; it does not any more. + +`plan.built` also keeps `marketId` and `borrower` as human-readable extras. They are for an operator +reading a single line — grouping keys on `id`. + +`id` identifies a **position**, and one position now yields several candidates (one per activated +collateral slot, and a matured-and-unhealthy slot in both open modes). The per-**candidate** key is +therefore `(id, collateralIndex, postMaturityMode)`, and both discriminators are carried on every +per-candidate event, the `@repo/swaps` quote events included. Two exceptions, deliberately: +`send.revert_streak` is per position because the streak spans whichever siblings reverted, and +`probe.*` events are per venue pair rather than per position — several positions in one market share +one probe. + ## Important Operational Notes - The liquidator gate checks the Executor address, not the EOA, because `liquidate` is called by the diff --git a/bots/midnight-liquidation/src/quotes.ts b/bots/midnight-liquidation/src/quotes.ts index 99c050db..97c0e256 100644 --- a/bots/midnight-liquidation/src/quotes.ts +++ b/bots/midnight-liquidation/src/quotes.ts @@ -98,7 +98,12 @@ export function composeQuoting(deps: { // The operator opt-out applies to the RAW collateral — midnight has no per-collateral config // file, so this is its escape hatch from the auto-unwrap path too. if (excluded(collateral.token)) { - logger.info('quote.excluded_collateral', { collateral: collateral.token }) + logger.info('quote.excluded_collateral', { + id: label, + collateralIndex: plan.collateralIndex, + postMaturityMode: plan.postMaturityMode, + collateral: collateral.token + }) return { kind: 'no_config', firmCalls: 0 } } @@ -112,7 +117,13 @@ export function composeQuoting(deps: { // mode by surplus, so the LIF is not recoverable from `postMaturityMode` or from chain time. minAcceptableAmountOut: plan.impliedRepaidUnits, // The tick's position label (`${id}:${borrower}`) — the correlation id join across quote logs. - id: label + id: label, + // Named exactly as the tick names them on its own events, so a join over both needs no + // normalization: one position emits several candidates under one `id`. + candidate: { + collateralIndex: plan.collateralIndex, + postMaturityMode: plan.postMaturityMode + } }) } } diff --git a/bots/midnight-liquidation/src/runner/tick.ts b/bots/midnight-liquidation/src/runner/tick.ts index f123ab44..0b036055 100644 --- a/bots/midnight-liquidation/src/runner/tick.ts +++ b/bots/midnight-liquidation/src/runner/tick.ts @@ -196,6 +196,7 @@ const LEVEL_BY_REASON: Record = { */ type SizedCandidate = { pair: LensInput + /** The position's {@link lensKey} — suppression-store key AND the `id` field of every log event. */ label: string out: LensOut plan: LiquidationPlan @@ -270,6 +271,7 @@ const sizeCandidates = (deps: { // them, because a matured-and-unhealthy slot is sized in BOTH modes and they are gated separately. for (const { reason, collateralIndex, headroom } of skips) { logger[LEVEL_BY_REASON[reason]]('plan.skipped', { + id: label, marketId: pair.id, borrower: pair.borrower, collateralIndex, @@ -395,9 +397,11 @@ const prepareRoutes = async (deps: { const resolved = await tryCatch(routing.resolveRoute(candidate.plan, candidate.out)) if (resolved.error) { logger.warn('route.unresolved', { + id: candidate.label, marketId: candidate.pair.id, borrower: candidate.pair.borrower, collateralIndex: candidate.plan.collateralIndex, + postMaturityMode: candidate.plan.postMaturityMode, detail: resolved.error.message }) } @@ -674,12 +678,13 @@ export async function runTick(deps: { counters.candidates = sized.length const skipPreselected = ( - dropped: readonly { pair: LensInput; plan: LiquidationPlan }[], + dropped: readonly { pair: LensInput; label: string; plan: LiquidationPlan }[], reason: 'probe_cap' | 'position_cap' ) => { counters.preselectSkipped += dropped.length for (const candidate of dropped) { logger.info('preselect.skipped', { + id: candidate.label, marketId: candidate.pair.id, borrower: candidate.pair.borrower, collateralIndex: candidate.plan.collateralIndex, @@ -768,6 +773,7 @@ export async function runTick(deps: { ) { counters.preselectSkipped += 1 logger.info('preselect.skipped', { + id: label, marketId: pair.id, borrower: pair.borrower, collateralIndex: liquidationPlan.collateralIndex, @@ -782,6 +788,9 @@ export async function runTick(deps: { // lines IS the record of what the bot worked and in what order, which is how the 31 Jul maturity // was reconstructed at all. logger.info('plan.built', { + id: label, + // Human-readable extras only: the pair `id` is built from, kept for an operator reading one + // line. Grouping keys on `id`. marketId: pair.id, borrower: pair.borrower, rank, @@ -803,7 +812,13 @@ export async function runTick(deps: { // (POSITION_LIQUIDATION_COOLDOWN_MS=0). if (cooldown.shouldSkip(label)) { counters.cooledDown += 1 - logger.info('cooldown.skip', { marketId: pair.id, borrower: pair.borrower }) + logger.info('cooldown.skip', { + id: label, + marketId: pair.id, + borrower: pair.borrower, + collateralIndex: liquidationPlan.collateralIndex, + postMaturityMode: liquidationPlan.postMaturityMode + }) continue } @@ -829,6 +844,7 @@ export async function runTick(deps: { counters.noSwapPath += 1 pendingCooldown.add(label) logger.info('config.no_swap_path', { + id: label, marketId: pair.id, borrower: pair.borrower, collateralIndex: liquidationPlan.collateralIndex, @@ -869,6 +885,7 @@ export async function runTick(deps: { // the route can clear `requiredRepay` and still be rejected, and an operator cannot tell // which rule fired without seeing the bar that was applied. logger.info('quote.unprofitable', { + id: label, marketId: pair.id, borrower: pair.borrower, collateralIndex: liquidationPlan.collateralIndex, @@ -891,10 +908,11 @@ export async function runTick(deps: { plan: liquidationPlan, swapPlan }) - // `collateralIndex` and `postMaturityMode` identify WHICH candidate this was: several entries - // per position share a (marketId, borrower), so without them two attempts on one position are - // indistinguishable in the log join. + // `id` joins the position across stages; `collateralIndex` and `postMaturityMode` identify WHICH + // candidate this was, since several entries per position share one `id` and without them two + // attempts on one position are indistinguishable in the log join. const fields = { + id: label, marketId: pair.id, borrower: pair.borrower, collateralIndex: liquidationPlan.collateralIndex, @@ -965,7 +983,10 @@ export async function runTick(deps: { // Only the crossing, so one stuck position is one warn per streak rather than one per tick // (two, when both its siblings revert) for as long as it stays stuck. if (streak.escalate === 'crossed') { + // No candidate discriminator: the streak is keyed by POSITION and spans whichever + // siblings reverted, so attributing it to one `(slot, mode)` would misreport it. logger.warn('send.revert_streak', { + id: label, marketId: pair.id, borrower: pair.borrower, reverts: streak.count, diff --git a/bots/midnight-liquidation/test/quotes.test.ts b/bots/midnight-liquidation/test/quotes.test.ts index 6548adf7..e7e6b983 100644 --- a/bots/midnight-liquidation/test/quotes.test.ts +++ b/bots/midnight-liquidation/test/quotes.test.ts @@ -201,6 +201,26 @@ describe('composeQuoting (Midnight lens-projection adapter)', () => { expect(selectOk?.fields?.id).toBe(LABEL) }) + it('threads the candidate discriminator, so two candidates of one position stay separable', async () => { + // The swaps-side gap BOTS-90 leaves otherwise: both candidates carry one `id`, so without the + // discriminator their `select.ok` rows are indistinguishable. + const events: { event: string; fields?: Record }[] = [] + const capturing: Logger = { + debug: () => {}, + info: (event, fields) => events.push({ event, fields }), + warn: () => {}, + error: () => {} + } + const { selector } = fakeSelector(['0x']) + const { quoteFor } = compose(selector, { logger: capturing }) + await quoteFor(PLAN, OUT, LABEL) + await quoteFor({ ...PLAN, postMaturityMode: true }, OUT, LABEL) + const rows = events.filter(e => e.event === 'select.ok') + expect(rows.map(e => e.fields?.id)).toEqual([LABEL, LABEL]) + expect(rows.map(e => e.fields?.collateralIndex)).toEqual([0, 0]) + expect(rows.map(e => e.fields?.postMaturityMode)).toEqual([false, true]) + }) + it('projects the plan break-even into the venue slippage it asks for', () => { // seizedAssets 1000 at price 1e36 -> reference 1000; break-even 800 -> 2000bps of allowance. Pins // that the adapter threads `impliedRepaidUnits` rather than leaving the floor unset. diff --git a/bots/midnight-liquidation/test/runner/tick.test.ts b/bots/midnight-liquidation/test/runner/tick.test.ts index 1232cfec..54d31718 100644 --- a/bots/midnight-liquidation/test/runner/tick.test.ts +++ b/bots/midnight-liquidation/test/runner/tick.test.ts @@ -3,7 +3,7 @@ import type { CooldownStore } from '@repo/bot-kit' import type { QuoteOutcome, SwapPlan, VenuePair } from '@repo/swaps' import type { Address, Hex } from 'viem' -import { createBackoff, createCooldownStore, TxSendError } from '@repo/bot-kit' +import { createBackoff, createCooldownStore, createPendingQueue, TxSendError } from '@repo/bot-kit' import { lensKey } from '@repo/utils' import { getAddress } from 'viem' import { describe, expect, it } from 'vitest' @@ -225,8 +225,12 @@ function runWith(opts: { routeCostBps?: Map /** Models an estimate taken from a ladder end: present in the curve, but not trustworthy. */ clampedRoutes?: boolean + /** Shared spy, so a caller can observe the tick's and the queue's events in ONE stream. */ + spy?: ReturnType + /** Replaces the stub `submit` — used to broadcast through a real pending queue. */ + submitWith?: (args: { label: string; blockNumber: bigint }) => Promise }) { - const { logger, events } = spyLogger() + const { logger, events } = opts.spy ?? spyLogger() const order: Address[] = [] let simulateCalls = 0 let submitCalls = 0 @@ -316,9 +320,10 @@ function runWith(opts: { opts.simulateResults?.[Math.min(simulateCalls - 1, opts.simulateResults.length - 1)] return sequenced ?? opts.simulateResult ?? { status: 'ok' } }, - submit: async () => { + submit: async args => { submitCalls += 1 if (opts.submitThrows) throw opts.submitThrows + if (opts.submitWith) return opts.submitWith(args) return opts.submitOutcome ?? { sent: true } }, backoff, @@ -1631,4 +1636,49 @@ describe('runTick', () => { expect(typeof end?.fields?.durationMs).toBe('number') }) }) + describe('position join key', () => { + it('emits one id that joins plan.built to the queue tx.sent', async () => { + // BOTS-90's acceptance criterion as a test: grouping a maturity's events by `id` must not split + // one position. Broadcast through the REAL queue, since the split was between the tick's field + // name and the queue's — a stubbed submit cannot see it. + const spy = spyLogger() + const queue = createPendingQueue({ + send: async () => ({ nonce: 7, txHash: `0x${'1'.repeat(64)}` }), + getReceipt: async () => null, + getBaseFee: async () => 1n, + maxFeeWei: 10n ** 18n, + logger: spy.logger + }) + await runWith({ + spy, + submitWith: ({ label, blockNumber }) => + queue.submit({ + request: { to: ROUTER, data: '0x' }, + label, + maxFeePerGas: 1000n, + maxPriorityFeePerGas: 1000n, + blockNumber + }) + }) + const planBuilt = spy.events.find(e => e.event === 'plan.built') + const txSent = spy.events.find(e => e.event === 'tx.sent') + expect(planBuilt?.fields?.id).toBe(LABEL) + expect(txSent?.fields?.id).toBe(planBuilt?.fields?.id) + }) + + it('gives one position two candidate rows sharing the id, separated by the discriminator', async () => { + // Multi-collateral: `id` alone can no longer identify a ROW, so the pair + // (collateralIndex, postMaturityMode) is what has to separate the position's alternatives. + const { events } = await runWith({ + out: twoSlots(), + quoteOutcome: { kind: 'failed', reason: 'no_route' } + }) + const built = events.filter(e => e.event === 'plan.built') + expect(built.map(e => e.fields?.id)).toEqual([LABEL, LABEL]) + expect(built.map(e => [e.fields?.collateralIndex, e.fields?.postMaturityMode])).toEqual([ + [1, false], + [0, false] + ]) + }) + }) }) diff --git a/docs/decisions/TIB-2026-05-28-midnight-liquidation-bot.md b/docs/decisions/TIB-2026-05-28-midnight-liquidation-bot.md index 308ff4c2..8afad3ce 100644 --- a/docs/decisions/TIB-2026-05-28-midnight-liquidation-bot.md +++ b/docs/decisions/TIB-2026-05-28-midnight-liquidation-bot.md @@ -744,6 +744,13 @@ are deferred to v1 (see Future Considerations). _Update (2026-07-14): the BetterStack log-forwarding half is now implemented additively — see [TIB-2026-07-14-betterstack-log-forwarding](./TIB-2026-07-14-betterstack-log-forwarding.md)._ +_Update (2026-08-31, BOTS-90): the field names in the catalogue above are historical. Every +position-scoped event now carries the position as a single **`id`** field valued +`lensKey(marketId, borrower)` — including the `tx.*` events, whose `label` field was renamed to `id`. +`marketId` / `borrower` survive on `plan.built` as human-readable extras only. See +[TIB-2026-08-28-midnight-send-shortfall-classification](./TIB-2026-08-28-midnight-send-shortfall-classification.md) +and the bots' READMEs._ + ## Security - **Private key handling.** `LIQUIDATOR_PRIVATE_KEY` read from env once at startup, never logged, diff --git a/docs/decisions/TIB-2026-08-28-midnight-send-shortfall-classification.md b/docs/decisions/TIB-2026-08-28-midnight-send-shortfall-classification.md index 82e99a01..e07e9857 100644 --- a/docs/decisions/TIB-2026-08-28-midnight-send-shortfall-classification.md +++ b/docs/decisions/TIB-2026-08-28-midnight-send-shortfall-classification.md @@ -27,8 +27,9 @@ what remains unobserved. A future reader who needs one thing from this document distinction, not the numbers. Source data throughout: BetterStack source `2607569` (`t384553.bot_liquidation_midnight`), window -2026-08-28 15:00:00–15:10:00 UTC, joined to Base receipts. Log joins key on `label` — the `tx.*` -events carry the position identity in `label`, not `id`. +2026-08-28 15:00:00–15:10:00 UTC, joined to Base receipts. The joins below key on `label`, which is +what the `tx.*` events carried at the time; BOTS-90 has since renamed that emitted field to `id` +(same value), so reproducing this analysis on a current window keys on `id` instead. ## Goals / Non-Goals @@ -329,9 +330,16 @@ protocol switch buried in it. its ramp does not trip it. The count is emitted alongside so a count-based rule can be calibrated from real data later if one turns out to be wanted. -- Dashboards joining sends to outcomes must key on `label`; `tx.*` events do not carry `id` **at the - time of this decision**. BOTS-90 normalizes that to a single `id` field across `plan.*`, `quote.*`, - `select.*`, `simulate.*` and `tx.*` in both liquidators — expect this line to be superseded. +- ~~Dashboards joining sends to outcomes must key on `label`; `tx.*` events do not carry `id` **at the + time of this decision**.~~ **Superseded by BOTS-90**, which landed in the same release train: every + position-scoped event in both liquidators — `plan.*`, `preselect.*`, `route.*`, `cooldown.*`, + `config.*`, `quote.*`, `unwrap.*`, `select.*`, `simulate.*`, `send.*`, `tx.*`, `queue.*`, `nonce.*` — + now carries the position as **`id`**, valued `lensKey(marketId, borrower)` (both halves lowercased), + so `GROUP BY id` needs no normalization. The queue's emitted `label` field is renamed to `id`; its + `SubmitArgs.label` input and in-flight map keys keep the name, because they are behavioral. That + rename also changes the `tx.*` log schema of `vault-v1-reallocation`, `vault-v2-reallocation` and + `midnight-crossed-books`, which pass a vault address or a market id as the same key. Join sends to + outcomes on `id`. - `tick.end` counter identities **do change**, in two places, and a dashboard summing them must be updated with the release rather than after it: - `notSent === sendRefused + sendReverted + sendRejected` is new. `sendReverted` is the exempt class diff --git a/packages/bot-kit/src/queue/pending-queue.ts b/packages/bot-kit/src/queue/pending-queue.ts index 6e98d029..0f344760 100644 --- a/packages/bot-kit/src/queue/pending-queue.ts +++ b/packages/bot-kit/src/queue/pending-queue.ts @@ -53,6 +53,14 @@ export type GetConsumedNonce = () => Promise /** What a caller hands {@link PendingQueue.submit}. */ export type SubmitArgs = { request: TxRequest + /** + * Opaque key this send is tracked and deduplicated under — a liquidator's `lensKey` position key, a + * reallocation bot's vault address, a resolver's market id. **Behavioral**, and deliberately still + * named `label` while the queue LOGS it as `id`: the two names are not a half-finished rename. + * {@link PendingQueue.inflightLabels} membership is tested against this exact string every tick, so + * changing its name or its casing here would silently miss a live entry and let a second + * nonce-consuming send go out for a position already in flight. + */ label: string maxFeePerGas: bigint maxPriorityFeePerGas: bigint @@ -249,7 +257,7 @@ export function createPendingQueue({ // Latched by a prior hashless send: skip until the next `onBlock` clears it. The signer has // rolled its cursor back, so broadcasting again now would race that rollback. if (sendAborted) { - logger.warn('tx.send_aborted', { label: args.label }) + logger.warn('tx.send_aborted', { id: args.label }) return { sent: false, reason: 'refused' } } // Nothing in flight → reconcile the cursor with chain before claiming a nonce. A failed sync @@ -261,7 +269,7 @@ export function createPendingQueue({ if (syncNonce && pending.size === 0) { const synced = await tryCatch(syncNonce()) if (synced.error) { - logger.warn('nonce.sync_failed', { label: args.label, reason: revertReason(synced.error) }) + logger.warn('nonce.sync_failed', { id: args.label, reason: revertReason(synced.error) }) return { sent: false, reason: 'refused' } } if (nonceHoleLow !== null) clearNonceHole('sync') @@ -271,7 +279,7 @@ export function createPendingQueue({ // above on an empty queue; here it only fires while other entries remain in flight. The onBlock // sweep clears it once the chain consumes past the hole. if (nonceHoleLow !== null) { - logger.warn('queue.nonce_hole', { label: args.label, nonce: nonceHoleLow }) + logger.warn('queue.nonce_hole', { id: args.label, nonce: nonceHoleLow }) return { sent: false, reason: 'refused' } } const sent = await tryCatch( @@ -285,7 +293,7 @@ export function createPendingQueue({ const executionRevert = isExecutionRevert(sent.error) const selector = revertSelector(sent.error) logger.warn('tx.submit_failed', { - label: args.label, + id: args.label, reason: revertReason(sent.error), executionRevert, ...(selector ? { selector } : {}), @@ -319,7 +327,7 @@ export function createPendingQueue({ attempt: 0 }) logger.info('tx.sent', { - label: args.label, + id: args.label, nonce, txHash, maxFee: args.maxFeePerGas, @@ -340,7 +348,7 @@ export function createPendingQueue({ settle(entry, blockNumber) latchNonceHole(entry.nonce) logger.warn('tx.dropped', { - label: entry.label, + id: entry.label, nonce: entry.nonce, txHash: entry.txHash, reason: 'max_bump_attempts' @@ -357,7 +365,7 @@ export function createPendingQueue({ settle(entry, blockNumber) latchNonceHole(entry.nonce) logger.warn('tx.dropped', { - label: entry.label, + id: entry.label, nonce: entry.nonce, txHash: entry.txHash, reason: 'fee_ceiling' @@ -373,7 +381,7 @@ export function createPendingQueue({ settle(entry, blockNumber) latchNonceHole(entry.nonce) logger.warn('tx.dropped', { - label: entry.label, + id: entry.label, nonce: entry.nonce, txHash: entry.txHash, reason: 'reverts_on_replace', @@ -382,7 +390,7 @@ export function createPendingQueue({ } else { entry.attempt += 1 logger.warn('tx.replace_failed', { - label: entry.label, + id: entry.label, nonce: entry.nonce, txHash: entry.txHash, attempt: entry.attempt, @@ -398,7 +406,7 @@ export function createPendingQueue({ entry.submittedAtBlock = blockNumber entry.attempt += 1 logger.info('tx.bumped', { - label: entry.label, + id: entry.label, nonce: entry.nonce, oldHash, newHash: replaced.data.txHash, @@ -477,14 +485,14 @@ export function createPendingQueue({ settle(entry, blockNumber) if (receipt.status === 'success') { logger.info('tx.confirmed', { - label: entry.label, + id: entry.label, nonce: entry.nonce, txHash: entry.txHash, blockNumber: receipt.blockNumber }) } else { logger.warn('tx.reverted', { - label: entry.label, + id: entry.label, nonce: entry.nonce, txHash: entry.txHash, blockNumber: receipt.blockNumber @@ -498,7 +506,7 @@ export function createPendingQueue({ } } catch (error) { logger.warn('tx.onblock_error', { - label: entry.label, + id: entry.label, nonce: entry.nonce, txHash: entry.txHash, reason: revertReason(error) @@ -516,7 +524,7 @@ export function createPendingQueue({ const entry = pending.get(nonce) if (!entry) return false logger.warn('tx.dropped', { - label: entry.label, + id: entry.label, nonce: entry.nonce, txHash: entry.txHash, reason diff --git a/packages/bot-kit/test/queue/pending-queue.test.ts b/packages/bot-kit/test/queue/pending-queue.test.ts index c1524eb5..239681e1 100644 --- a/packages/bot-kit/test/queue/pending-queue.test.ts +++ b/packages/bot-kit/test/queue/pending-queue.test.ts @@ -378,7 +378,7 @@ describe('drop', () => { const dropped = events.find(e => e.event === 'tx.dropped') expect(dropped?.level).toBe('warn') expect(dropped?.fields).toMatchObject({ - label: 'market:borrower', + id: 'market:borrower', nonce: 7, txHash: hashOf(1), reason: 'nonce_consumed' @@ -416,7 +416,7 @@ describe('nonce-consumed reconciliation', () => { await queue.onBlock(1n) expect(queue.size).toBe(0) expect(events.find(e => e.event === 'tx.dropped')?.fields).toMatchObject({ - label: 'market:borrower', + id: 'market:borrower', nonce: 7, txHash: hashOf(1), reason: 'nonce_consumed' @@ -558,9 +558,7 @@ describe('nonce-hole latch', () => { expect(await ctx.submit('c', 6n)).toEqual({ sent: false, reason: 'refused' }) expect(ctx.sends.length).toBe(sendsAfterDrop) // no new broadcast expect(ctx.queue.size).toBe(1) - expect(ctx.events.some(e => e.event === 'queue.nonce_hole' && e.fields?.label === 'c')).toBe( - true - ) + expect(ctx.events.some(e => e.event === 'queue.nonce_hole' && e.fields?.id === 'c')).toBe(true) // The chain now consumes past the dropped nonce (7 mined → count 8 > hole high 7): latch clears. ctx.consumedRef.value = 8 await ctx.queue.onBlock(7n) diff --git a/packages/swaps/src/quoting.ts b/packages/swaps/src/quoting.ts index e8594cb4..fd36716e 100644 --- a/packages/swaps/src/quoting.ts +++ b/packages/swaps/src/quoting.ts @@ -141,8 +141,24 @@ export type QuoteRequest = { tokenInDecimals?: number /** The position's correlation id — threaded into log events only, never parsed. */ id?: string + /** + * Which of the position's candidates this request is for, when a protocol yields several from one + * {@link QuoteRequest.id} (Midnight's `(collateral slot, mode)` alternatives). Spread verbatim onto + * this package's log events, under the same never-parsed contract as `id`: without it two candidates + * of one position emit rows a query cannot tell apart. Fields must be named exactly as the calling + * bot names them on its own events, since a join spanning both must not normalize. + */ + candidate?: Readonly> } +/** + * The correlation fields every event in this package carries: the position's join key plus whatever + * discriminates the candidate within it. `id` is spread LAST of the two so a stray `candidate.id` + * cannot shadow the join key, and each event's own fields are spread after this so neither can shadow + * them. + */ +const correlationOf = (request: QuoteRequest) => ({ ...request.candidate, id: request.id }) + // Normalizes a venue adapter's Swap into the plan's final step: the spender becomes the step's // approval target and the venue-agnostic call fields carry over verbatim. function toStep(swap: Swap, tokenIn: Address, tokenOut: Address): SwapStep { @@ -439,7 +455,7 @@ async function tryResolveUnwraps( if (error || !resolution) { const reason = error instanceof QuoteError ? error.reason : 'api_error' logger.warn('unwrap.failed', { - id: request.id, + ...correlationOf(request), collateral: request.collateralToken, reason, detail: ensureError(error).message @@ -448,7 +464,7 @@ async function tryResolveUnwraps( } if (resolution.steps.length > 0) { logger.info('unwrap.resolved', { - id: request.id, + ...correlationOf(request), collateral: request.collateralToken, path: [...resolution.steps.map(step => step.tokenIn), resolution.token], amountIn: resolution.amountIn @@ -495,8 +511,8 @@ const swapFreePlan = (args: { }) ) { logger.warn('unwrap.bad_route', { + ...correlationOf(request), venue: path, - id: request.id, collateral: request.collateralToken, expected: resolution.amountIn, oracle: request.referenceAmountOut @@ -509,8 +525,8 @@ const swapFreePlan = (args: { // unrelated: a chain can clear `maxRouteImpactBps` and still land under the repay. if (resolution.amountIn < request.minAcceptableAmountOut) { logger.info('quote.floor_unmet', { + ...correlationOf(request), venue: path, - id: request.id, collateral: request.collateralToken, expected: resolution.amountIn, amountOutMinimum: resolution.amountIn, @@ -520,8 +536,8 @@ const swapFreePlan = (args: { return { kind: 'failed', reason: 'floor_unmet' } } logger.info('quote.ok', { + ...correlationOf(request), venue: path, - id: request.id, collateral: request.collateralToken, expected: resolution.amountIn, oracle: request.referenceAmountOut, @@ -638,13 +654,14 @@ export function composeMultiVenueQuoting(deps: { pair: VenuePair amountIn: bigint referenceAmountOut: bigint - id?: string + /** {@link correlationOf}'s output for the request being quoted. */ + correlation: Record }): Promise<{ order: Venue[]; estimates: Map; trusted: boolean }> => { - const { pair, amountIn, referenceAmountOut, id } = args + const { pair, amountIn, referenceAmountOut, correlation } = args const { error: probeError } = await tryCatch(refresh(pair)) if (probeError) { logger.warn('probe.error', { - id, + ...correlation, collateral: pair.collateral, loan: pair.loan, detail: probeError.message @@ -655,7 +672,12 @@ export function composeMultiVenueQuoting(deps: { const estimates = new Map(ranked.map(estimate => [estimate.venue, estimate])) const order = [...estimates.keys(), ...venues.filter(venue => !estimates.has(venue))] if (ranked.length === 0) { - logger.info('select.cold_default', { collateral: pair.collateral, loan: pair.loan, order }) + logger.info('select.cold_default', { + ...correlation, + collateral: pair.collateral, + loan: pair.loan, + order + }) } const trusted = ranked.length > 0 && @@ -666,7 +688,8 @@ export function composeMultiVenueQuoting(deps: { return { async quoteFor(request) { - const { collateralToken, loanToken, referenceAmountOut, id } = request + const { collateralToken, loanToken, referenceAmountOut } = request + const correlation = correlationOf(request) const unwrapped = await tryResolveUnwraps(unwrappers, request, executor, logger) if ('outcome' in unwrapped) return { ...unwrapped.outcome, firmCalls: 0 } @@ -687,7 +710,7 @@ export function composeMultiVenueQuoting(deps: { pair, amountIn: resolution.amountIn, referenceAmountOut, - id + correlation }) // A trusted curve ranked every enabled venue on exactly the axis `bad_route` and `floor_unmet` // measure, so once the winner has been quoted those two say nothing a runner-up would change and @@ -720,8 +743,8 @@ export function composeMultiVenueQuoting(deps: { if (outcome.kind === 'quote_failed') { lastReason = outcome.reason logger.warn('quote.failed', { + ...correlation, venue, - id, collateral: collateralToken, reason: lastReason, detail: outcome.detail @@ -734,8 +757,8 @@ export function composeMultiVenueQuoting(deps: { // every venue misses it for every candidate on every block until the incentive catches up. // That is the ordinary early-ramp shape, not an anomaly to page on. logger.info('quote.floor_unmet', { + ...correlation, venue, - id, collateral: collateralToken, expected: outcome.swap.expectedAmountOut, amountOutMinimum: outcome.swap.amountOutMinimum, @@ -748,8 +771,8 @@ export function composeMultiVenueQuoting(deps: { if (outcome.kind === 'bad_route') { lastReason = 'bad_route' logger.warn('quote.route_quality_failed', { + ...correlation, venue, - id, collateral: collateralToken, expected: outcome.swap.expectedAmountOut, oracle: referenceAmountOut @@ -758,8 +781,8 @@ export function composeMultiVenueQuoting(deps: { continue } logger.info('select.ok', { + ...correlation, venue, - id, collateral: collateralToken, expected: outcome.swap.expectedAmountOut, oracle: referenceAmountOut, diff --git a/packages/utils/src/helpers/deployless-batch-lens.ts b/packages/utils/src/helpers/deployless-batch-lens.ts index 009c6579..a9e185ea 100644 --- a/packages/utils/src/helpers/deployless-batch-lens.ts +++ b/packages/utils/src/helpers/deployless-batch-lens.ts @@ -30,6 +30,11 @@ export const MAX_INITCODE_SIZE = 49_152 /** * Stable per-pair key for a batch-lens result map. `id` widens to {@link Hex} so both a market id * (bytes32) and an address-shaped id key uniformly. + * + * Also **the** position join key across a liquidator's log events, emitted verbatim as their `id` + * field. Both halves are lowercased here so that grouping a maturity's events by `id` needs no + * normalization in the query; a call site that rebuilds the composite itself reintroduces the + * checksum-versus-lowercase split this exists to remove. */ export function lensKey(id: Hex, borrower: Address): string { return `${id.toLowerCase()}:${borrower.toLowerCase()}` From 564ebdd7e72233e397c5acae1c3b441d4051bc6d Mon Sep 17 00:00:00 2001 From: Hayden Shively Date: Mon, 31 Aug 2026 12:10:07 -0500 Subject: [PATCH 2/2] address review of the position join key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four documentation claims the code did not deliver, and one duplicated comment: - The READMEs globbed `unwrap.*` / `queue.*` / `nonce.*` as carrying `id`. Both now enumerate the position-scoped events exactly and list what is scoped to a tick, a venue pair, the queue, or the process instead — a glob over a mixed set is what makes a `GROUP BY id` drop rows into a null bucket. - `unwrap.preview_reverted` / `unwrap.preview_zero` genuinely were position-scoped and carried no `id`: `previewRedeem` is amount-dependent, so unlike the `asset()` probe it is not memoized and fires per candidate. Correlation now threads through `Unwrapper.resolve`, which is also why `resolveRoute` takes the label. - The claimed `probe.*` exception was false for `probe.error`, which does carry the correlation. Named the four pair-scoped events instead. - `correlationOf`'s TSDoc claimed every event in `@repo/swaps`; the venue selector and both unwrappers carry none. - The `plan.built` comment was duplicated verbatim in both ticks and narrated two self-evident fields. The READMEs are its one home. `docs/CONVENTIONS.md` gains the rule itself, so `lensKey`'s TSDoc, the two READMEs and `SubmitArgs.label` point at one place rather than restating it four times. The three other queue consumers now say what their `tx.*.id` is: a checksummed vault address, or a market id. It joins to nothing in those bots, which key their own events on `vault`. Tests: `@repo/swaps` pins the discriminator spread, the non-shadowing of `id`, and that correlation reaches an unwrap hop; `@repo/bot-kit` pins the emitted field name on `tx.sent` and `tx.confirmed` in the package that owns the schema. Co-Authored-By: Claude Opus 5 (1M context) --- bots/blue-liquidation/README.md | 31 +++++++--- bots/blue-liquidation/src/quotes.ts | 1 - bots/blue-liquidation/src/runner/tick.ts | 2 - bots/midnight-crossed-books/README.md | 6 ++ bots/midnight-liquidation/README.md | 39 ++++++++---- bots/midnight-liquidation/src/quotes.ts | 28 +++++---- bots/midnight-liquidation/src/runner/tick.ts | 15 +++-- bots/vault-v1-reallocation/README.md | 5 ++ bots/vault-v2-reallocation/README.md | 5 ++ docs/CONVENTIONS.md | 8 +++ packages/bot-kit/src/queue/pending-queue.ts | 11 ++-- .../bot-kit/test/queue/pending-queue.test.ts | 14 +++++ packages/swaps/src/quoting.ts | 18 +++--- packages/swaps/src/unwrappers/erc4626.ts | 6 +- packages/swaps/src/unwrappers/resolve.ts | 19 +++++- packages/swaps/test/quoting.test.ts | 60 +++++++++++++++++++ .../src/helpers/deployless-batch-lens.ts | 6 +- 17 files changed, 211 insertions(+), 63 deletions(-) diff --git a/bots/blue-liquidation/README.md b/bots/blue-liquidation/README.md index f5510d4c..ca2a375e 100644 --- a/bots/blue-liquidation/README.md +++ b/bots/blue-liquidation/README.md @@ -262,15 +262,28 @@ the nonce from `getTransactionCount('pending')`. ### Log correlation -Every position-scoped event — `plan.*`, `cooldown.*`, `config.*`, `quote.*`, `unwrap.*`, `select.*`, -`simulate.*`, `tx.*`, `queue.*`, `nonce.*` — carries the position in one field, **`id`**, whose value -is `lensKey(marketId, borrower)`: the two halves joined by `:` with both lowercased. So a window's -events group into one row per position with **no normalization in the query** (`GROUP BY id`). `tx.*` -used to name the same string `label`; it does not any more. `plan.built` also keeps `marketId` and -`borrower` as human-readable extras — for reading a single line, not for grouping. - -A Blue market has exactly one collateral, so one position is one candidate: unlike -`bots/midnight-liquidation`, `id` alone identifies a row and no candidate discriminator is emitted. +Every **position-scoped** event carries the position in one field, **`id`**, whose value is +`lensKey(marketId, borrower)`: the two halves joined by `:` with both lowercased. So a window's events +group into one row per position with **no normalization in the query** (`GROUP BY id`). `tx.*` used to +name the same string `label`; it does not any more. The full set: + +`plan.built`, `cooldown.skip`, `config.no_swap_path`, `quote.excluded_collateral`, `unwrap.failed`, +`unwrap.resolved`, `unwrap.bad_route`, `unwrap.preview_reverted`, `unwrap.preview_zero`, +`quote.floor_unmet`, `quote.ok`, `quote.failed`, `quote.route_quality_failed`, `probe.error`, +`select.cold_default`, `select.ok`, `simulate.ok`, `simulate.revert`, `tx.send_aborted`, +`tx.submit_failed`, `tx.sent`, `tx.bumped`, `tx.confirmed`, `tx.reverted`, `tx.dropped`, +`tx.replace_failed`, `tx.onblock_error`, `nonce.sync_failed`, `queue.nonce_hole`. + +`plan.built` also keeps `marketId` and `borrower` as human-readable extras — for reading a single +line, not for grouping. A Blue market has exactly one collateral, so one position is one candidate: +unlike `bots/midnight-liquidation`, `id` alone identifies a row and no candidate discriminator is +emitted. + +Everything else is scoped to something other than a position and carries **no** `id`, by design — +per tick (`discover.*`, `lens.read`, `tick.end`, `tick.error`, `block.new`), per venue pair +(`probe.venue_error`, `probe.refreshed`), queue-wide (`queue.nonce_hole_cleared`, +`queue.maintenance_failed`, `reconcile.failed`), or process/config (`startup`, `shutdown`, +`quoting.*`, `discovery.*`, `signer.*`, `heartbeat.*`, `runner.*`, `watcher.error`, `pendle.*`). ## Testing diff --git a/bots/blue-liquidation/src/quotes.ts b/bots/blue-liquidation/src/quotes.ts index 386d7533..98c11fb1 100644 --- a/bots/blue-liquidation/src/quotes.ts +++ b/bots/blue-liquidation/src/quotes.ts @@ -64,7 +64,6 @@ export function composeQuoting(deps: { // Break-even, straight off the plan: the loan assets `liquidate` will pull for this seize, // including the shares round-trip Blue settles through. minAcceptableAmountOut: plan.impliedRepaidAssets, - // The tick's position label (`${id}:${borrower}`) — the correlation id join across quote logs. id: label }) } diff --git a/bots/blue-liquidation/src/runner/tick.ts b/bots/blue-liquidation/src/runner/tick.ts index c6a0fe80..06cc5959 100644 --- a/bots/blue-liquidation/src/runner/tick.ts +++ b/bots/blue-liquidation/src/runner/tick.ts @@ -147,8 +147,6 @@ export async function runTick(deps: { counters.planned += 1 logger.info('plan.built', { id: label, - // Human-readable extras only: the pair `id` is built from, kept for an operator reading one - // line. Grouping keys on `id`. marketId: id, borrower: pair.borrower, seizedAssets: liquidationPlan.seizedAssets diff --git a/bots/midnight-crossed-books/README.md b/bots/midnight-crossed-books/README.md index 1834ca8b..520aeaec 100644 --- a/bots/midnight-crossed-books/README.md +++ b/bots/midnight-crossed-books/README.md @@ -112,6 +112,12 @@ label or a manual production workflow dispatch and the `crossed-books-prod` GitH Each GitHub Environment defines `RAILWAY_PROJECT_ID` as a variable and `RAILWAY_TOKEN` as a secret; bot runtime secrets remain on Railway. +## Observability + +The shared queue identifies a tracked transaction by the key this bot hands it, under the field `id` +(it was `label`). This bot's key is the **market id**, so `tx.*.id` is a market rather than a +position — it does not join to a liquidator's `id`, which is `marketId:borrower`. + ## Test ```sh diff --git a/bots/midnight-liquidation/README.md b/bots/midnight-liquidation/README.md index 625c949a..d1474ec9 100644 --- a/bots/midnight-liquidation/README.md +++ b/bots/midnight-liquidation/README.md @@ -590,22 +590,39 @@ that tick instead of counting a hashless transaction as submitted. ### Log Correlation -Every position-scoped event — `plan.*`, `preselect.*`, `route.*`, `cooldown.*`, `config.*`, `quote.*`, -`unwrap.*`, `select.*`, `simulate.*`, `send.*`, `tx.*`, `queue.*`, `nonce.*` — carries the position in -one field, **`id`**, whose value is `lensKey(marketId, borrower)`: the two halves joined by `:` with -both lowercased. So a maturity's events group into one row per position with **no normalization in the -query** (`GROUP BY id`). `tx.*` used to name the same string `label`; it does not any more. +Every **position-scoped** event carries the position in one field, **`id`**, whose value is +`lensKey(marketId, borrower)`: the two halves joined by `:` with both lowercased. So a maturity's +events group into one row per position with **no normalization in the query** (`GROUP BY id`). `tx.*` +used to name the same string `label`; it does not any more. The full set: + +`plan.skipped`, `plan.built`, `preselect.skipped`, `route.unresolved`, `cooldown.skip`, +`config.no_swap_path`, `quote.excluded_collateral`, `quote.unprofitable`, `unwrap.failed`, +`unwrap.resolved`, `unwrap.bad_route`, `unwrap.preview_reverted`, `unwrap.preview_zero`, +`quote.floor_unmet`, `quote.ok`, `quote.failed`, `quote.route_quality_failed`, `probe.error`, +`select.cold_default`, `select.ok`, `simulate.ok`, `simulate.revert`, `send.revert_streak`, +`tx.send_aborted`, `tx.submit_failed`, `tx.sent`, `tx.bumped`, `tx.confirmed`, `tx.reverted`, +`tx.dropped`, `tx.replace_failed`, `tx.onblock_error`, `nonce.sync_failed`, `queue.nonce_hole`. `plan.built` also keeps `marketId` and `borrower` as human-readable extras. They are for an operator reading a single line — grouping keys on `id`. -`id` identifies a **position**, and one position now yields several candidates (one per activated +`id` identifies a **position**, and one position yields several candidates (one per activated collateral slot, and a matured-and-unhealthy slot in both open modes). The per-**candidate** key is -therefore `(id, collateralIndex, postMaturityMode)`, and both discriminators are carried on every -per-candidate event, the `@repo/swaps` quote events included. Two exceptions, deliberately: -`send.revert_streak` is per position because the streak spans whichever siblings reverted, and -`probe.*` events are per venue pair rather than per position — several positions in one market share -one probe. +therefore `(id, collateralIndex, postMaturityMode)`, and both discriminators ride on every +per-candidate event, the `@repo/swaps` quote events included. `send.revert_streak` deliberately +carries none: the streak is keyed by position and spans whichever siblings reverted, so attributing it +to one `(slot, mode)` would misreport it. + +Everything else is scoped to something other than a position and carries **no** `id`, by design — +don't group it by one: + +- **per tick** — `discover.*`, `lens.read`, `probe.warmed`, `tick.end`, `tick.error`, `block.new` +- **per venue pair** — `probe.warm_failed`, `probe.venue_error`, `probe.refreshed` (several positions + in one market share a probe) +- **queue-wide** — `queue.nonce_hole_cleared`, `queue.maintenance_failed`, `reconcile.failed` (a + condition that would have refused any position) +- **process / config** — `startup`, `shutdown`, `quoting.*`, `markets.*`, `prices.*`, `discovery.*`, + `signer.*`, `heartbeat.*`, `runner.*`, `watcher.error`, `pendle.*` ## Important Operational Notes diff --git a/bots/midnight-liquidation/src/quotes.ts b/bots/midnight-liquidation/src/quotes.ts index 97c0e256..faa6adf7 100644 --- a/bots/midnight-liquidation/src/quotes.ts +++ b/bots/midnight-liquidation/src/quotes.ts @@ -17,6 +17,15 @@ import type { LensOut } from './state/lens.sol' import { expectedLoanOut } from './execution/swap-step' +/** + * What separates one position's `(slot, mode)` alternatives in a log row, for `@repo/swaps`' + * `QuoteRequest.candidate`. Correlation only; the quoting layer never reads it. + */ +const candidateOf = (plan: LiquidationPlan) => ({ + collateralIndex: plan.collateralIndex, + postMaturityMode: plan.postMaturityMode +}) + /** * The Midnight-shaped adapter over `@repo/swaps`' {@link composeMultiVenueQuoting}: keeps the * `(plan, out)` signature the tick consumes and projects the lens output into the package's plain @@ -56,7 +65,8 @@ export function composeQuoting(deps: { */ resolveRoute: ( plan: LiquidationPlan, - out: LensOut + out: LensOut, + label: string ) => Promise<{ pair: VenuePair; amountIn: bigint } | null> } { const { selector, excludeCollaterals, logger, executor, unwrappers, ...rest } = deps @@ -74,14 +84,15 @@ export function composeQuoting(deps: { }) return { - async resolveRoute(plan, out) { + async resolveRoute(plan, out, label) { const collateral = out.market.collateralParams[plan.collateralIndex] if (!collateral || excluded(collateral.token)) return null const resolution = await resolveUnwraps(unwrappers, { token: collateral.token, amountIn: plan.seizedAssets, executor, - stopToken: out.market.loanToken + stopToken: out.market.loanToken, + correlation: { id: label, ...candidateOf(plan) } }) if (isAddressEqual(resolution.token, out.market.loanToken)) return null return { @@ -100,8 +111,7 @@ export function composeQuoting(deps: { if (excluded(collateral.token)) { logger.info('quote.excluded_collateral', { id: label, - collateralIndex: plan.collateralIndex, - postMaturityMode: plan.postMaturityMode, + ...candidateOf(plan), collateral: collateral.token }) return { kind: 'no_config', firmCalls: 0 } @@ -116,14 +126,8 @@ export function composeQuoting(deps: { // the plan was sized at. Read rather than recomputed — the matured-and-unhealthy branch picks a // mode by surplus, so the LIF is not recoverable from `postMaturityMode` or from chain time. minAcceptableAmountOut: plan.impliedRepaidUnits, - // The tick's position label (`${id}:${borrower}`) — the correlation id join across quote logs. id: label, - // Named exactly as the tick names them on its own events, so a join over both needs no - // normalization: one position emits several candidates under one `id`. - candidate: { - collateralIndex: plan.collateralIndex, - postMaturityMode: plan.postMaturityMode - } + candidate: candidateOf(plan) }) } } diff --git a/bots/midnight-liquidation/src/runner/tick.ts b/bots/midnight-liquidation/src/runner/tick.ts index 0b036055..617ff666 100644 --- a/bots/midnight-liquidation/src/runner/tick.ts +++ b/bots/midnight-liquidation/src/runner/tick.ts @@ -330,9 +330,14 @@ type TickRouting = { * path that already ends in the loan token. `null` is treated as unknown cost rather than as free: * only sizing's `swapFree` flag asserts that no route is needed. * - * Async, which is the entire reason phase A.5 exists — see {@link sizeCandidates}. + * Async, which is the entire reason phase A.5 exists — see {@link sizeCandidates}. `label` is + * correlation only: it is what the unwrap hops log their diagnostics under. */ - resolveRoute: (plan: LiquidationPlan, out: LensOut) => Promise + resolveRoute: ( + plan: LiquidationPlan, + out: LensOut, + label: string + ) => Promise /** * Fills the probe cache for one pair. A cold refresh is one indicative venue call per ladder rung per * venue on the isolated probe client, so it is driven only for pairs that have a sized candidate. @@ -394,7 +399,9 @@ const prepareRoutes = async (deps: { states.set(candidate, { kind: 'no_route' }) continue } - const resolved = await tryCatch(routing.resolveRoute(candidate.plan, candidate.out)) + const resolved = await tryCatch( + routing.resolveRoute(candidate.plan, candidate.out, candidate.label) + ) if (resolved.error) { logger.warn('route.unresolved', { id: candidate.label, @@ -789,8 +796,6 @@ export async function runTick(deps: { // was reconstructed at all. logger.info('plan.built', { id: label, - // Human-readable extras only: the pair `id` is built from, kept for an operator reading one - // line. Grouping keys on `id`. marketId: pair.id, borrower: pair.borrower, rank, diff --git a/bots/vault-v1-reallocation/README.md b/bots/vault-v1-reallocation/README.md index 5c8bd13f..48b3ab6b 100644 --- a/bots/vault-v1-reallocation/README.md +++ b/bots/vault-v1-reallocation/README.md @@ -173,6 +173,11 @@ stamped on every line. Key events: `startup`, `allocator.missing_role`, `vault.i `duration_ms`), and the shared bot-kit `tx.*` / `signer.balance` / `block.new` events. BetterStack shipping and heartbeat are opt-in via the env vars above. +The shared queue identifies a tracked transaction by the key this bot hands it, under the field `id` +(it was `label`). This bot's key is the **vault address, checksummed** — not a liquidator's +`lensKey` — so `tx.*.id` does not join to this bot's own events, which key on `vault`. Compare the two +case-insensitively. + ## Testing ```sh diff --git a/bots/vault-v2-reallocation/README.md b/bots/vault-v2-reallocation/README.md index 5a9ee732..6314b461 100644 --- a/bots/vault-v2-reallocation/README.md +++ b/bots/vault-v2-reallocation/README.md @@ -177,6 +177,11 @@ stamped on every line. Key events: `startup`, `allocator.missing_role`, `realloc `vault.error`, per-pass `tick.end` counters, and the shared bot-kit `tx.*` / `signer.balance` / `block.new` events. BetterStack shipping and heartbeat are opt-in via the env vars above. +The shared queue identifies a tracked transaction by the key this bot hands it, under the field `id` +(it was `label`). This bot's key is the **vault address, checksummed** — not a liquidator's +`lensKey` — so `tx.*.id` does not join to this bot's own events, which key on `vault`. Compare the two +case-insensitively. + ## Testing ```sh diff --git a/docs/CONVENTIONS.md b/docs/CONVENTIONS.md index bb5c9fc1..4626e5e3 100644 --- a/docs/CONVENTIONS.md +++ b/docs/CONVENTIONS.md @@ -50,6 +50,14 @@ - **Logging**: Log errors appropriately for debugging and monitoring. Prefer structured logs with enough context (bot name, operation, relevant inputs) that an operator can answer "what did the bot do and why?" a day later. +- **One join key per subject**: every event scoped to the same subject carries that subject under + **one** field name, with one shape and one casing, produced by a shared helper rather than + assembled at each call site — so grouping needs no normalization in the query. The liquidators' + subject is a position and the field is `id` (see `lensKey` in `@repo/utils`); a subject that + subdivides adds a discriminator beside the key rather than changing it, named identically in every + package that emits it. A key that is also **behavioral** (a map key, a dedupe set) keeps whatever + name that role gave it — `PendingQueue`'s `SubmitArgs.label` is logged as `id` but stays `label` in + the API, because its value is compared, not just displayed. Never re-derive the key at a call site. - **Promises**: Use `tryCatch` from `@repo/utils` to handle promise throws. ### Environment Variables diff --git a/packages/bot-kit/src/queue/pending-queue.ts b/packages/bot-kit/src/queue/pending-queue.ts index 0f344760..99cbf0cf 100644 --- a/packages/bot-kit/src/queue/pending-queue.ts +++ b/packages/bot-kit/src/queue/pending-queue.ts @@ -55,11 +55,10 @@ export type SubmitArgs = { request: TxRequest /** * Opaque key this send is tracked and deduplicated under — a liquidator's `lensKey` position key, a - * reallocation bot's vault address, a resolver's market id. **Behavioral**, and deliberately still - * named `label` while the queue LOGS it as `id`: the two names are not a half-finished rename. - * {@link PendingQueue.inflightLabels} membership is tested against this exact string every tick, so - * changing its name or its casing here would silently miss a live entry and let a second - * nonce-consuming send go out for a position already in flight. + * reallocation bot's vault address, a resolver's market id. Logged as `id`; behavioral here, and the + * divergence is deliberate: {@link PendingQueue.inflightLabels} membership is tested against this + * exact string every tick, so renaming it or normalizing its casing would silently miss a live entry + * and let a second nonce-consuming send go out for a position already in flight. */ label: string maxFeePerGas: bigint @@ -123,7 +122,7 @@ export type PendingQueue = { readonly size: number snapshot(): { nonce: number; txHash: Hex; attempt: number }[] /** - * Labels (`${id}:${borrower}`) the tick must NOT re-submit — its backpressure set. Covers + * {@link SubmitArgs.label}s the tick must NOT re-submit — its backpressure set. Covers * currently-pending txs AND, when `settledCooldownBlocks` is set, positions whose tx settled * within that many blocks. The cooldown matters when sends and reads use different RPCs: a tx * confirms on the send RPC before the (laggy) read RPC reflects the cleared position, so without diff --git a/packages/bot-kit/test/queue/pending-queue.test.ts b/packages/bot-kit/test/queue/pending-queue.test.ts index 239681e1..4bc44475 100644 --- a/packages/bot-kit/test/queue/pending-queue.test.ts +++ b/packages/bot-kit/test/queue/pending-queue.test.ts @@ -323,6 +323,20 @@ describe('createPendingQueue', () => { expect(queue.inflightLabels().has('market:borrower')).toBe(true) }) + it('emits the tracking key as `id` on the send and on the settlement', async () => { + // The field name is the schema every dashboard joins on, and this package owns it. The behavioral + // key keeps the name `label` on the way IN — see SubmitArgs.label. + const { logger, events } = captureLogger() + const { queue } = setup({ + logger, + getReceipt: async () => ({ status: 'success', blockNumber: 10n }) + }) + await submitOne(queue, 0n) + await queue.onBlock(1n) + expect(events.find(e => e.event === 'tx.sent')?.fields?.id).toBe('market:borrower') + expect(events.find(e => e.event === 'tx.confirmed')?.fields?.id).toBe('market:borrower') + }) + it('keeps a confirmed label in the backpressure set for the cooldown, then releases it', async () => { const { queue } = setup({ getReceipt: async () => ({ status: 'success', blockNumber: 10n }) }) await submitOne(queue, 0n) diff --git a/packages/swaps/src/quoting.ts b/packages/swaps/src/quoting.ts index fd36716e..e8f32a93 100644 --- a/packages/swaps/src/quoting.ts +++ b/packages/swaps/src/quoting.ts @@ -146,16 +146,19 @@ export type QuoteRequest = { * {@link QuoteRequest.id} (Midnight's `(collateral slot, mode)` alternatives). Spread verbatim onto * this package's log events, under the same never-parsed contract as `id`: without it two candidates * of one position emit rows a query cannot tell apart. Fields must be named exactly as the calling - * bot names them on its own events, since a join spanning both must not normalize. + * bot names them on its own events, since a join spanning both must not normalize — and must avoid + * the names those events already use (`venue`, `collateral`, `loan`, `reason`, `expected`, `oracle`, + * `floor`, `order`, `detail`, `path`, `amountIn`, `amountOutMinimum`, `minOutSource`), each of which + * wins the spread and would drop the discriminator from the row. */ candidate?: Readonly> } /** - * The correlation fields every event in this package carries: the position's join key plus whatever - * discriminates the candidate within it. `id` is spread LAST of the two so a stray `candidate.id` - * cannot shadow the join key, and each event's own fields are spread after this so neither can shadow - * them. + * The correlation fields every event {@link composeMultiVenueQuoting} emits carries — and every hop it + * drives through {@link Unwrapper.resolve} — being the position's join key plus whatever discriminates + * the candidate within it. `id` is spread LAST of the two so a stray `candidate.id` cannot shadow the + * join key, and each event's own fields are spread after this so neither can shadow them. */ const correlationOf = (request: QuoteRequest) => ({ ...request.candidate, id: request.id }) @@ -449,7 +452,8 @@ async function tryResolveUnwraps( token: request.collateralToken, amountIn: request.amountIn, executor, - stopToken: request.loanToken + stopToken: request.loanToken, + correlation: correlationOf(request) }) ) if (error || !resolution) { @@ -655,7 +659,7 @@ export function composeMultiVenueQuoting(deps: { amountIn: bigint referenceAmountOut: bigint /** {@link correlationOf}'s output for the request being quoted. */ - correlation: Record + correlation: ReturnType }): Promise<{ order: Venue[]; estimates: Map; trusted: boolean }> => { const { pair, amountIn, referenceAmountOut, correlation } = args const { error: probeError } = await tryCatch(refresh(pair)) diff --git a/packages/swaps/src/unwrappers/erc4626.ts b/packages/swaps/src/unwrappers/erc4626.ts index 385d93b9..b470f919 100644 --- a/packages/swaps/src/unwrappers/erc4626.ts +++ b/packages/swaps/src/unwrappers/erc4626.ts @@ -80,7 +80,7 @@ export function createErc4626Unwrapper(deps: { client: Client; logger: QuoteLogg return { kind: 'erc4626', - async resolve({ token, amountIn, executor }) { + async resolve({ token, amountIn, executor, correlation }) { const underlying = await underlyingFor(token) if (underlying === null) return null @@ -97,11 +97,11 @@ export function createErc4626Unwrapper(deps: { client: Client; logger: QuoteLogg }) } catch (error) { if (!isContractLevelFailure(error)) throw error - logger.warn('unwrap.preview_reverted', { unwrapper: 'erc4626', token }) + logger.warn('unwrap.preview_reverted', { ...correlation, unwrapper: 'erc4626', token }) return null } if (previewed === 0n) { - logger.warn('unwrap.preview_zero', { unwrapper: 'erc4626', token }) + logger.warn('unwrap.preview_zero', { ...correlation, unwrapper: 'erc4626', token }) return null } diff --git a/packages/swaps/src/unwrappers/resolve.ts b/packages/swaps/src/unwrappers/resolve.ts index 0b5d5d21..8cab2db1 100644 --- a/packages/swaps/src/unwrappers/resolve.ts +++ b/packages/swaps/src/unwrappers/resolve.ts @@ -17,6 +17,12 @@ export type Unwrapper = { token: Address amountIn: bigint executor: Address + /** + * Correlation fields for whichever candidate drove this hop, spread onto any per-hop log event. + * Never parsed, and never a reason to behave differently — a hop's outcome depends on the token + * and the amount alone. + */ + correlation?: Readonly> }) => Promise<{ step: SwapStep; expectedAmountOut: bigint; amountOutMinimum: bigint } | null> } @@ -46,9 +52,16 @@ export type UnwrapResolution = { */ export async function resolveUnwraps( unwrappers: readonly Unwrapper[], - args: { token: Address; amountIn: bigint; executor: Address; stopToken: Address } + args: { + token: Address + amountIn: bigint + executor: Address + stopToken: Address + /** Forwarded verbatim to every hop — see {@link Unwrapper.resolve}. */ + correlation?: Readonly> + } ): Promise { - const { executor, stopToken } = args + const { executor, stopToken, correlation } = args const steps: SwapStep[] = [] let token = args.token let amountIn = args.amountIn @@ -58,7 +71,7 @@ export async function resolveUnwraps( let advanced = false for (const unwrapper of unwrappers) { - const result = await unwrapper.resolve({ token, amountIn, executor }) + const result = await unwrapper.resolve({ token, amountIn, executor, correlation }) if (!result) continue // A hop that doesn't change the token can never terminate — treat it as "does not apply". if (isAddressEqual(result.step.tokenIn, result.step.tokenOut)) continue diff --git a/packages/swaps/test/quoting.test.ts b/packages/swaps/test/quoting.test.ts index 1a5558ad..23b4ad0c 100644 --- a/packages/swaps/test/quoting.test.ts +++ b/packages/swaps/test/quoting.test.ts @@ -494,6 +494,66 @@ describe('composeMultiVenueQuoting', () => { // Every case here reads the FALLBACK derivation, so the curve is clamped throughout: the percentage // under test is the one derived against the oracle reference, which is what a venue is asked for when // the probe has no trustworthy prediction of its output. +describe('correlation fields', () => { + // These carry the position join key and the candidate discriminator across the pipeline stages, so + // a maturity's events group without normalization. Correlation only — never parsed, never branched on. + const correlated: QuoteRequest = { + ...REQUEST, + id: '0xabc:0x1111111111111111111111111111111111111111', + candidate: { collateralIndex: 2, postMaturityMode: true } + } + + it('spreads the candidate discriminator beside the id on every quote event', async () => { + const { quoteFor, events } = composeMulti( + ['0x'], + [{ venue: '0x', expectedOut: 1000n }], + multiHttp({ '0x': zeroxBody('1000') }) + ) + await quoteFor(correlated) + const selectOk = events.find(e => e.event === 'select.ok') + expect(selectOk?.fields).toMatchObject({ + id: correlated.id, + collateralIndex: 2, + postMaturityMode: true + }) + }) + + it('never lets the discriminator shadow the join key', async () => { + // The discriminator is caller-supplied, so a key collision must not be able to rewrite `id` — + // that would resurrect exactly the split this field set exists to remove. + const { quoteFor, events } = composeMulti( + ['0x'], + [{ venue: '0x', expectedOut: 1000n }], + multiHttp({ '0x': zeroxBody('1000') }) + ) + await quoteFor({ ...correlated, candidate: { ...correlated.candidate, id: 'spoofed' } }) + expect(events.find(e => e.event === 'select.ok')?.fields?.id).toBe(correlated.id) + }) + + it('reaches the unwrap hops, so a per-hop diagnostic is attributable', async () => { + const seen: (Record | undefined)[] = [] + const probe: Unwrapper = { + kind: 'probe', + resolve: async ({ correlation }) => { + seen.push(correlation) + return null + } + } + const { quoteFor } = composeMulti( + ['0x'], + [{ venue: '0x', expectedOut: 1000n }], + multiHttp({ '0x': zeroxBody('1000') }), + { unwrappers: [probe] } + ) + await quoteFor(correlated) + expect(seen[0]).toEqual({ + collateralIndex: 2, + postMaturityMode: true, + id: correlated.id + }) + }) +}) + describe('economic min-out floor', () => { // Captures the slippage each venue was asked for, which is the aggregators' ONLY min-out lever. const capturingHttp = (body: unknown) => { diff --git a/packages/utils/src/helpers/deployless-batch-lens.ts b/packages/utils/src/helpers/deployless-batch-lens.ts index a9e185ea..625b8dd1 100644 --- a/packages/utils/src/helpers/deployless-batch-lens.ts +++ b/packages/utils/src/helpers/deployless-batch-lens.ts @@ -31,10 +31,8 @@ export const MAX_INITCODE_SIZE = 49_152 * Stable per-pair key for a batch-lens result map. `id` widens to {@link Hex} so both a market id * (bytes32) and an address-shaped id key uniformly. * - * Also **the** position join key across a liquidator's log events, emitted verbatim as their `id` - * field. Both halves are lowercased here so that grouping a maturity's events by `id` needs no - * normalization in the query; a call site that rebuilds the composite itself reintroduces the - * checksum-versus-lowercase split this exists to remove. + * Also the liquidators' log join key (emitted verbatim as `id` — see "One join key per subject" in + * `docs/CONVENTIONS.md`), which is why both halves are lowercased here rather than at each call site. */ export function lensKey(id: Hex, borrower: Address): string { return `${id.toLowerCase()}:${borrower.toLowerCase()}`