diff --git a/CHANGELOG.md b/CHANGELOG.md index 188b498..7727eac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.16.0] - 2026-09-08 + +### Added +- `npm run clear-reorg-halt -- --reason "..."` clears a durable REORG_HALT marker after verifying the database is intact, recording the reason and checks as a REORG_HALT_CLEARED event. +- The health surface reports `reorg_halt_cleared_at` and `reorg_halt_cleared_reason` once a halt has been cleared. + +### Fixed +- A node still in initial block download with its tip below the stored tip is waited on instead of being reconciled as a reorg. +- The health and status payloads carry `node_catching_up` while that wait runs, so `xchain-node ps` can show it instead of a stopped height. +- A node-tip gap deeper than the dispenser safe-depth window is refused before the first delete, with no durable halt and no resync owed. + ## [0.15.0] - 2026-09-07 ### Changed diff --git a/README.md b/README.md index 0871276..9a92047 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ # XChain Platform Decoder

- Version - Tests + Version + Tests Node License

@@ -95,9 +95,8 @@ check CI runs across the vendored copies. These four names configure the shim itself. The fleet deploy path carries them into the container: `xchain-node` forwards any of them set in the module config store or in the deploy host's environment (`ModuleService.resolveObservabilityEnv`), -and the validator compose files under `claude/deploy/testnet-validators/` name -them outright. Nothing is fabricated when neither source sets one, so these -defaults hold on an unconfigured box: +and the validator compose files name them outright. Nothing is fabricated when +neither source sets one, so these defaults hold on an unconfigured box: | Variable | Default | Effect | |---|---|---| @@ -114,7 +113,7 @@ defaults hold on an unconfigured box: | `npm run migrate` | Apply pending database migrations (auto + manual; `--file ` scopes to specific migration(s)) | | `npm run ci` | The full no-external-services gate: unit, security, smoke, regression, chaos, and a 100-iteration fuzz pass (about a minute) | | `npm run test:smoke` | Smoke tests (58 tests, no external services) | -| `npm run test:unit` | Unit tests (1,450 tests, no external services) | +| `npm run test:unit` | Unit tests (1,549 tests, no external services) | | `npm run test:security` | Security tests (83 tests, no external services) | | `npm run test:integration` | Integration tests (30 tests; brings up its own throwaway regtest node and MariaDB, requires Docker) | | `npm run test:e2e` | End-to-end tests (72 tests; brings up its own throwaway regtest node and MariaDB on separate ports, requires Docker) | diff --git a/package-lock.json b/package-lock.json index eaf29cd..9a4b230 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "xchain-decoder", - "version": "0.15.0", + "version": "0.16.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "xchain-decoder", - "version": "0.15.0", + "version": "0.16.0", "license": "AGPL-3.0-or-later", "dependencies": { "axios": "^1.18.1", diff --git a/package.json b/package.json index 709ced7..91646cf 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "xchain-decoder", "description": "xchain-decoder decodes XChain platform transactions from a given blockchain and populates a database with the decoded data.", - "version": "0.15.0", + "version": "0.16.0", "license": "AGPL-3.0-or-later", "repository": { "type": "git", @@ -26,6 +26,7 @@ "scripts": { "api": "node ./src/api.js", "migrate": "node ./src/migrate.js", + "clear-reorg-halt": "node ./src/clear-reorg-halt.js", "test": "mocha --timeout 5000 --require ./test/unit/setup.js 'test/unit/**/*.test.js' --exit", "coverage": "c8 --reporter=text --reporter=html --include 'src/**/*.js' mocha --timeout 5000 --require ./test/unit/setup.js 'test/unit/**/*.test.js' --exit", "coverage:check": "c8 --check-coverage --lines 87.8 --statements 87.8 --branches 85.4 --functions 77.2 --reporter=text-summary --include 'src/**/*.js' mocha --timeout 5000 --require ./test/unit/setup.js 'test/unit/**/*.test.js' --exit", diff --git a/src/XChainDecoder.js b/src/XChainDecoder.js index 808ec5f..27787ee 100644 --- a/src/XChainDecoder.js +++ b/src/XChainDecoder.js @@ -113,6 +113,19 @@ const SYNCED_THRESHOLD = 3 //Maximum blocks behind to be synced // Purging deeper is the conservative direction (rows are merely retained longer // before hard-purge; expiry semantics and action evaluation are unchanged). const DISPENSER_EXPIRE_SAFE_DEPTH = 126 // 120 (deepest undo window, LTC and DOGE) + 6 margin + +// Whether a getblockchaininfo reply says the node is still in initial block +// download. While it is, a node tip BELOW the stored tip is not a rollback: the +// node has simply not yet validated blocks this database already holds (an +// operator's fresh mainnet node, a reindex, a node restored behind a decoder that +// followed another endpoint). Reconciling against that tip deletes valid blocks +// to the safe-depth ceiling and writes a durable halt for a reorg that never +// happened; the right move is to wait until the node passes the stored tip and +// let the forward hash compare decide. Strict === true: an absent field (an +// older node, a trimmed proxy) keeps the pre-existing behaviour. +function nodeStillCatchingUp(info){ + return !!info && info["initialblockdownload"] === true +} // There is deliberately no DISPENSER_CLOSE_DELAY twin of the indexer's here: the decoder // does not mirror dispenser cancels, so it never needs to close a row at the height the // indexer's DISPENSER_CLOSE fires. Reintroducing a closing mirror would need that pinned @@ -426,6 +439,15 @@ class XChainDecoder { // rollback and the bootstrap gate finds nothing to refuse on. this.reorgHaltMarkerPersisted = null this._reorgHaltProbeInFlight = null + + // Non-null only while the parse loop is waiting out a node in initial block + // download whose tip sits below our stored tip (see the wait branch in + // start()). That wait is otherwise indistinguishable from a wedge on every + // health surface: the height stops moving and nothing says why. Published + // verbatim as node_catching_up so `xchain-node ps` can name the wait. + // Shape: { node_height, stored_height, since } where since is the ISO + // timestamp the CURRENT wait began, held fixed until it ends. + this.nodeCatchingUp = null } async sleep(ms) { @@ -639,6 +661,10 @@ class XChainDecoder { this.reorgHalted = !!(marker && marker.halted) this.reorgHaltReason = (marker && marker.reason) || null this.reorgHaltAt = (marker && marker.at) || null + // An operator clear (db.clearReorgHalt) supersedes the halt; surface + // when and why so a cleared database still tells its history. + this.reorgHaltClearedAt = (marker && marker.cleared_at) || null + this.reorgHaltClearedReason = (marker && marker.cleared_reason) || null this.reorgHaltCheckedAt = now // A marker this probe just READ is durable by observation, whatever the // write that produced it reported. Raised here and never cleared here: @@ -676,6 +702,8 @@ class XChainDecoder { halted: !!this.reorgHalted, reason: this.reorgHaltReason || null, at: this.reorgHaltAt || null, + cleared_at: this.reorgHaltClearedAt || null, + cleared_reason: this.reorgHaltClearedReason || null, checked_at: this.reorgHaltCheckedAt || null, marker_persisted: (this.reorgHaltMarkerPersisted === null || this.reorgHaltMarkerPersisted === undefined) ? null : !!this.reorgHaltMarkerPersisted @@ -2010,6 +2038,35 @@ class XChainDecoder { // always passes the freshly-refreshed tip. if (nodeTip != null && lastBlockIndex > nodeTip){ await assertWithinSafeDepth(lastBlockIndex) + + // This branch knows its depth up front: every stored height above the + // node tip is a delete. When that alone (on top of what is already + // rolled back) would cross the ceiling, refuse NOW, before the first + // delete, and WITHOUT the durable halt: nothing has been rolled back + // past the window, so nothing is lost and no resync is owed. The + // ceiling check above stays the authority once deletes have happened; + // this only stops a run that is doomed from its first block from + // spending the whole window to find that out (an operator's mainnet + // node 2666 blocks behind lost 126 valid blocks and forty hours to + // exactly that, 2026-09-07). Tagged so the parse loop can wait on it + // instead of exiting into a restart loop. + const aboveTip = lastBlockIndex - nodeTip + const alreadyRolledBack = priorDepth + blocksDeleted.length + if (alreadyRolledBack + aboveTip > DISPENSER_EXPIRE_SAFE_DEPTH){ + const msg = "verifyReorg: the node's tip (" + nodeTip + ") is " + aboveTip + + " blocks below the stored tip (" + lastBlockIndex + "), which" + + (alreadyRolledBack > 0 ? " with " + alreadyRolledBack + " block(s) already rolled back" : "") + + " exceeds the dispenser safe-depth window (DISPENSER_EXPIRE_SAFE_DEPTH=" + + DISPENSER_EXPIRE_SAFE_DEPTH + "). Refusing before any further delete: nothing has been " + + "rolled back past the window, no REORG_HALT marker was written and this database needs " + + "no resync. Either the node is still catching up (wait for it to pass " + lastBlockIndex + + ") or it was rolled back below this database's tip (operator action)." + // Not logged here: the parse loop retries this every poll and logs + // the refusal once per transition; other callers let it escape. + const err = new Error(msg) + err.tipBelowStoredTip = true + throw err + } try { // Pass the block hash so the delete and its REORG audit marker commit // atomically; see deleteBlockByIndex for the durability rationale. @@ -2315,6 +2372,12 @@ class XChainDecoder { let nodeSyncedProblem = false + // Node-tip-below-ours latches, one line per transition each: the node is + // still in initial block download (wait, never reconcile), or the gap is + // too deep to reconcile and verifyReorg refused before deleting (wait, + // keep running, say so once). + let nodeCatchingUpProblem = false + let tipBelowStoredTipRefused = false // Wrong-tier endpoint latch, same shape as nodeSyncedProblem: the refusal // repeats every 3-second retry, so log it on the transition only. @@ -2493,6 +2556,15 @@ class XChainDecoder { continue } + // The usual end of an IBD wait: the node's tip reached our height, so the + // tip-regression branch below is simply never entered again and the + // in-branch clear cannot fire. Without this the finished wait would stay + // on every health payload for the life of the process. The log latch is + // deliberately NOT cleared here: it speaks only for the branch below. + if (this.nodeCatchingUp && lastProcessedBlockIndex <= this.blockchainInfoLastBlock){ + this.nodeCatchingUp = null + } + if (lastProcessedBlockIndex > this.blockchainInfoLastBlock){ if (lastProcessedBlockIndex == this.startBlockIndex - 1){ // Benign: we have processed nothing yet and the node simply @@ -2502,6 +2574,35 @@ class XChainDecoder { continue } + // A node still in initial block download has not validated up to + // our height yet; its tip below ours is a node catching up, not a + // rollback. Wait for it to pass the stored tip, then the forward + // hash compare below decides whether anything diverged. Measured + // on an operator's fresh BTC mainnet node 2026-09-07: reconciling + // here rolled back 126 valid blocks, hit the safe-depth ceiling, + // wrote the durable halt and crash-looped 279 times over a reorg + // that never happened. The wait is also published as + // this.nodeCatchingUp (health payloads: node_catching_up), because a + // silent wait is indistinguishable from a wedge: the height stops + // moving and every surface still reads green. Both heights are + // re-read each poll; `since` is carried over so it keeps naming the + // instant THIS wait began. + if (nodeStillCatchingUp(lastBlockchainInfo)){ + if (!nodeCatchingUpProblem){ + this.logWarn("The last processed block height ("+lastProcessedBlockIndex+") is greater than the last block from the node ("+this.blockchainInfoLastBlock+"), but the node reports initialblockdownload=true: it is still catching up, not rolled back. Waiting for it to pass "+lastProcessedBlockIndex+" instead of reconciling; the hash compare decides then.") + } + const since = (this.nodeCatchingUp && this.nodeCatchingUp.since) || new Date().toISOString() + this.nodeCatchingUp = { node_height: this.blockchainInfoLastBlock, stored_height: lastProcessedBlockIndex, since } + nodeCatchingUpProblem = true + await this.sleep(5000) + continue + } + if (nodeCatchingUpProblem){ + this.log("The node has left initial block download with its tip ("+this.blockchainInfoLastBlock+") still below the last processed block ("+lastProcessedBlockIndex+"); treating the gap as a rollback from here on.") + nodeCatchingUpProblem = false + } + this.nodeCatchingUp = null + // The node's tip has dropped BELOW our last-processed height (deep // reorg, node rollback, or restart onto a shorter/different chain). // The forward hash-compare reorg path (below) is unreachable in this @@ -2513,9 +2614,30 @@ class XChainDecoder { // deterministic height compare, then walks the hash-compare back to // the fork point. blockchainInfoLastBlock was just refreshed above, so // the tip is current. - this.log("The last processed block height ("+lastProcessedBlockIndex+") is greater than the last block from the node ("+this.blockchainInfoLastBlock+"). Reconciling orphan blocks...") + if (!tipBelowStoredTipRefused){ + this.log("The last processed block height ("+lastProcessedBlockIndex+") is greater than the last block from the node ("+this.blockchainInfoLastBlock+"). Reconciling orphan blocks...") + } await this.db.endTransaction() - await this.verifyReorg(this.blockchainInfoLastBlock) + try { + await this.verifyReorg(this.blockchainInfoLastBlock) + } catch (err){ + // A gap too deep to reconcile, refused BEFORE any delete (nothing + // rolled back, no durable halt). Exiting here would only restart + // into the same refusal; stay up, say it once, and re-check the + // tip every poll so a node that is merely catching up (without + // reporting IBD) resolves it on its own and a real rollback stays + // visible on the status surface as node_height below the tip. + if (err && err.tipBelowStoredTip){ + if (!tipBelowStoredTipRefused){ + this.logError(err.message) + } + tipBelowStoredTipRefused = true + await this.sleep(5000) + continue + } + throw err + } + tipBelowStoredTipRefused = false // Re-clamp: a deep reorg can empty the blocks table, causing // getLastBlockIndex() to return -1 and nextBlockHeight to become 0 // on a nonzero-start network. Clamp here, the same as the pre-loop guard. @@ -3668,6 +3790,7 @@ module.exports.compiledPushSize = compiledPushSize module.exports.OP_RETURN_PUSH_OVERHEAD = OP_RETURN_PUSH_OVERHEAD // Exported so a regression test can pin it >= the deepest per-chain reorg window. module.exports.DISPENSER_EXPIRE_SAFE_DEPTH = DISPENSER_EXPIRE_SAFE_DEPTH +module.exports.nodeStillCatchingUp = nodeStillCatchingUp // Exported so the funding-fee-output collision regression test can assert attributed // funding outputs are stored at vout + FUNDING_VOUT_BASE (never colliding with real vouts). module.exports.FUNDING_VOUT_BASE = FUNDING_VOUT_BASE diff --git a/src/api.js b/src/api.js index 37d7b66..6d781ec 100644 --- a/src/api.js +++ b/src/api.js @@ -192,6 +192,9 @@ function registerLiveRoute(app, decoder, isDecoderRunning){ reorg_halted: reorgHalt.halted === true, reorg_halt_reason: reorgHalt.reason || null, reorg_halted_at: reorgHalt.at || null, + // { node_height, stored_height, since } while the parse loop is waiting out + // a node in initial block download below our tip, null otherwise. + node_catching_up: (decoder && decoder.nodeCatchingUp) || null, // A frozen node tip, reported but deliberately NOT gating. isStalled() // returns false while the tip is stale on purpose: restarting the container // cannot fix an upstream node outage, and gating on it re-opens the @@ -393,7 +396,7 @@ async function startApi(){ // restart-loop a service that is doing useful work while fixing nothing (the // marker survives restarts and is only cleared by a resync). Report it as its // own field instead, and let the operator/watchdog act on it. - let reorgHalt = { halted: false, reason: null, at: null, checked_at: null } + let reorgHalt = { halted: false, reason: null, at: null, cleared_at: null, cleared_reason: null, checked_at: null } if (dbOk && typeof decoder.checkReorgHalt === 'function'){ try { reorgHalt = await decoder.checkReorgHalt() } catch (e) { noteProbeFailure('reorg_halt', 'rpc:health', e) } } @@ -409,6 +412,13 @@ async function startApi(){ reorg_halted: reorgHalt.halted, reorg_halt_reason: reorgHalt.reason, reorg_halted_at: reorgHalt.at, + // { node_height, stored_height, since } while the parse loop is waiting + // out a node in initial block download below our tip, null otherwise. + node_catching_up: (decoder && decoder.nodeCatchingUp) || null, + // Set once an operator cleared a halt (db.clearReorgHalt); null while a + // halt is live or none was ever recorded. + reorg_halt_cleared_at: reorgHalt.cleared_at || null, + reorg_halt_cleared_reason: reorgHalt.cleared_reason || null, reorg_halt_checked_at: reorgHalt.checked_at, ...syncStatus, lastProcessedBlock: syncStatus.last_processed_block, @@ -516,6 +526,9 @@ async function startApi(){ reorg_halted: reorgHalt.halted, reorg_halt_reason: reorgHalt.reason, reorg_halted_at: reorgHalt.at, + // { node_height, stored_height, since } while the parse loop is waiting out + // a node in initial block download below our tip, null otherwise. + node_catching_up: (decoder && decoder.nodeCatchingUp) || null, // Ships beside the boolean, never without it. "Not halted" is only an answer // if something looked, and the probe is fail-soft: its state starts at // not-halted with checked_at null, so a decoder that has NEVER completed a diff --git a/src/clear-reorg-halt.js b/src/clear-reorg-halt.js new file mode 100644 index 0000000..0b5f926 --- /dev/null +++ b/src/clear-reorg-halt.js @@ -0,0 +1,171 @@ +/********************************************************************* + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * + * XChain Decoder - audited clear of a durable REORG_HALT marker + * + * node src/clear-reorg-halt.js --reason "" [--force] [--dry-run] + * (under xchain-node: `xchain-node clear-reorg-halt --reason "..."`) + * + * verifyReorg writes the REORG_HALT marker when a rollback crossed the dispenser + * safe-depth window: soft-expired dispenser rows below that depth were already + * hard-purged and cannot be resurrected, so the database MAY have lost + * money-bearing dispenser state. Until now the only recovery was a full resync, + * even for a database that never held a dispenser (a pre-launch mainnet decoder, + * measured 2026-09-07: forty hours on an operator's hardware to replace a database + * nothing had been purged from). + * + * This tool clears the marker WITHOUT deleting it: it writes a REORG_HALT_CLEARED + * row that carries the operator's reason, the checks that passed and the halt it + * supersedes, and db.readReorgHaltState lets the newest row decide. Preconditions: + * + * 1. no block is still missing above the tip (countReorgDeletesAboveTip == 0): + * the rolled-back range has been re-parsed. Cannot be forced; wait for the + * decoder to catch up. + * 2. the database holds no dispenser state and never decoded a DISPENSER + * action, so the purge cannot have lost anything. --force overrides this one + * for an operator who has compared the dispensers table against a known-good + * replica; the clear row records that it was forced. + * + * Reads DECODER_DB_* from the service environment (.env), like migrate.js. + * + ********************************************************************/ + +'use strict' + +const EXIT = { + OK: 0, + FAILED: 1, + USAGE: 2, + NOT_RESYNCED: 3, + DISPENSER_STATE: 4 +} + +const USAGE = 'usage: node src/clear-reorg-halt.js --reason "" [--force] [--dry-run]' + +function parseArgs(argv){ + const out = { reason: null, force: false, dryRun: false, help: false, bad: null } + for (let i = 0; i < argv.length; i++){ + const a = argv[i] + if (a === '--reason' || a === '-r'){ + const v = argv[i + 1] + if (v === undefined || v.startsWith('-')){ out.bad = a + ' requires a text argument'; return out } + out.reason = v; i++ + } else if (a.startsWith('--reason=')){ + out.reason = a.slice('--reason='.length) + } else if (a === '--force'){ + out.force = true + } else if (a === '--dry-run'){ + out.dryRun = true + } else if (a === '--help' || a === '-h'){ + out.help = true + } else { + out.bad = 'unknown argument ' + a; return out + } + } + return out +} + +// The whole decision, with the database and the output injected so it can be +// exercised without MariaDB. Returns the process exit code. +async function run({ db, argv = [], log = console.log, error = console.error }){ + const args = parseArgs(argv) + if (args.help){ log(USAGE); return EXIT.OK } + if (args.bad){ error('clear-reorg-halt: ' + args.bad + '\n' + USAGE); return EXIT.USAGE } + if (typeof args.reason !== 'string' || args.reason.trim().length < 8){ + error('clear-reorg-halt: --reason must say, in at least 8 characters, why this database is known good; it is recorded with the clear.\n' + USAGE) + return EXIT.USAGE + } + + const marker = await db.getReorgHaltMarker() + if (!marker.halted){ + log('clear-reorg-halt: no live REORG_HALT marker' + + (marker.cleared_at ? ' (last halt cleared ' + marker.cleared_at + ': ' + (marker.cleared_reason || 'no reason recorded') + ')' : '') + + '. Nothing to do.') + return EXIT.OK + } + log('clear-reorg-halt: live REORG_HALT marker' + (marker.at ? ' since ' + marker.at : '') + + (marker.reason ? ': ' + marker.reason : '')) + + // Check 1: the rollback has been re-synced. Not forceable: a halt with blocks + // still missing above the tip is a rollback in progress, and clearing it lets + // the next verifyReorg resume past the window. + const deletesAboveTip = await db.countReorgDeletesAboveTip() + if (deletesAboveTip > 0){ + error('clear-reorg-halt: REFUSED. ' + deletesAboveTip + ' block(s) rolled back above the current tip have not been re-parsed yet. ' + + 'Wait for the decoder to catch up past the halt height, then run this again. This check cannot be forced.') + return EXIT.NOT_RESYNCED + } + + // Check 2: nothing the purge could have lost. + const dispensers = await db.countDispensers() + const dispenserTxs = await db.hasDispenserTransactions() + const checks = { deletes_above_tip: deletesAboveTip, dispensers: dispensers, dispenser_transactions: dispenserTxs } + const dispenserClean = (dispensers === 0 && dispenserTxs === false) + if (!dispenserClean && !args.force){ + error('clear-reorg-halt: REFUSED. This database has held dispenser state (' + dispensers + ' dispenser row(s) now, ' + + (dispenserTxs ? 'DISPENSER actions decoded' : 'no DISPENSER action decoded') + '), so the purge the halt ' + + 'protects against may have dropped rows that a resync would recover. Compare the dispensers table against a ' + + 'known-good replica of this decoder; if it matches, run again with --force (the clear is recorded as forced). ' + + 'If it does not, resync from a known-good snapshot instead.') + return EXIT.DISPENSER_STATE + } + + const verdict = 'checks: rolled-back blocks above tip = 0; dispensers = ' + dispensers + '; DISPENSER actions decoded = ' + dispenserTxs + + (dispenserClean ? ' (clean)' : ' (FORCED by the operator)') + if (args.dryRun){ + log('clear-reorg-halt: dry run. ' + verdict + '. The marker would be cleared with reason: ' + args.reason.trim()) + return EXIT.OK + } + + const result = await db.clearReorgHalt({ reason: args.reason.trim(), checks: checks, forced: !dispenserClean }) + if (result.alreadyClear){ + log('clear-reorg-halt: the marker was cleared by someone else while this ran. Nothing to do.') + return EXIT.OK + } + if (!result.cleared){ + error('clear-reorg-halt: FAILED. The REORG_HALT_CLEARED row could not be written or read back; the halt is still live.') + return EXIT.FAILED + } + log('clear-reorg-halt: cleared. ' + verdict + '. Recorded as events.code=REORG_HALT_CLEARED with reason: ' + args.reason.trim() + + '. The decoder reports reorg_halted=false on its next probe (within a minute); the halt row itself is kept for the audit trail.') + return EXIT.OK +} + +async function main(){ + require('dotenv').config() + const Database = require('./db.js') + const host = process.env.DECODER_DB_HOST + const port = process.env.DECODER_DB_PORT + const name = process.env.DECODER_DB_NAME + const user = process.env.DECODER_DB_USER + const pass = process.env.DECODER_DB_PASS + if (!host || !name || !user){ + console.error('clear-reorg-halt: DECODER_DB_HOST / DECODER_DB_NAME / DECODER_DB_USER must be set (load the service .env).') + process.exit(EXIT.USAGE) + } + const db = new Database(host, port, name, user, pass) + let code = EXIT.FAILED + try { + code = await run({ db, argv: process.argv.slice(2) }) + } catch (err){ + console.error('clear-reorg-halt: FAILED: ' + ((err && err.stack) || err)) + } finally { + try { if (db.pool) await db.pool.end() } catch (_) {} + } + process.exitCode = code +} + +if (require.main === module) main() + +module.exports = { run, parseArgs, EXIT, USAGE } diff --git a/src/db.js b/src/db.js index 3a582a9..04731da 100644 --- a/src/db.js +++ b/src/db.js @@ -2672,13 +2672,43 @@ class Database { // is persisted as a REORG_HALT row in the events table (an existing durable // store); a full resync from a known-good snapshot rebuilds the schema and so // clears it, matching the recovery the abort message already demands. + // + // An operator can CLEAR a halt through clearReorgHalt (src/clear-reorg-halt.js, + // `xchain-node clear-reorg-halt`): that writes a REORG_HALT_CLEARED row carrying + // the reason and the checks that passed, and the NEWEST of the two codes decides. + // The halt row is never deleted, so the audit trail survives, and a later halt + // writes a newer REORG_HALT row that is live again. async isReorgHalted(){ - const query = `SELECT 1 FROM events WHERE code = 'REORG_HALT' LIMIT 1;` + return (await this.readReorgHaltState()).halted + } + + // The newest REORG_HALT / REORG_HALT_CLEARED row, ordered on the (code, id) + // index. Returns { halted, id, at, reason, cleared_at, cleared_reason }. + // Fail-closed: a halt row whose id or payload cannot be read still counts as + // live, because "we could not tell" must never reach a caller as "not halted". + async readReorgHaltState(){ + const query = `SELECT id, time, code, data FROM events WHERE code IN ('REORG_HALT', 'REORG_HALT_CLEARED') ORDER BY id DESC LIMIT 1;` + const none = { halted: false, id: null, at: null, reason: null, cleared_at: null, cleared_reason: null } let connection = await this.getConnection() const ownLease = (this.transactionConnection == null) try { const rows = await connection.query(query) - return Array.isArray(rows) ? rows.length > 0 : false + if (!Array.isArray(rows) || rows.length === 0) return none + const row = rows[0] + let payload = null + try { + payload = (typeof row.data === 'string') ? JSON.parse(row.data) : row.data + } catch (_) { + payload = null + } + const at = (payload && payload.at) ? payload.at : (row.time != null ? String(row.time) : null) + const reason = (payload && payload.reason) ? payload.reason : null + if (row.code === 'REORG_HALT_CLEARED'){ + return { ...none, cleared_at: at, cleared_reason: reason } + } + // Any other shape (the expected REORG_HALT, or a row whose code could not + // be read) is a live halt. + return { halted: true, id: (row.id != null ? row.id : null), at: at, reason: reason, cleared_at: null, cleared_reason: null } } finally { if (ownLease){ await connection.release() @@ -2686,6 +2716,67 @@ class Database { } } + // Number of rows in the dispensers table. The clear tool's first precondition: + // a database that holds no dispenser state cannot have lost any to the purge. + async countDispensers(){ + const query = `SELECT COUNT(*) AS n FROM dispensers;` + let connection = await this.getConnection() + const ownLease = (this.transactionConnection == null) + try { + const rows = await connection.query(query) + if (!Array.isArray(rows) || rows.length === 0 || rows[0].n == null) + throw new Error('countDispensers: the dispensers count could not be read') + return Number(rows[0].n) + } finally { + if (ownLease){ + await connection.release() + } + } + } + + // Whether this database has EVER decoded a DISPENSER action. A purged + // dispenser leaves no row behind, so an empty dispensers table alone does not + // prove nothing was purged; a database with no DISPENSER transaction at all does. + // LIMIT 1 stops at the first hit; a database with none scans the table once, + // which is acceptable for a one-off operator command. + async hasDispenserTransactions(){ + const query = `SELECT 1 FROM transactions WHERE data LIKE 'DISPENSER|%' LIMIT 1;` + let connection = await this.getConnection() + const ownLease = (this.transactionConnection == null) + try { + const rows = await connection.query(query) + if (!Array.isArray(rows)) throw new Error('hasDispenserTransactions: the DISPENSER probe could not be read') + return rows.length > 0 + } finally { + if (ownLease){ + await connection.release() + } + } + } + + // Audited operator clear of a live REORG_HALT marker. Writes a + // REORG_HALT_CLEARED row carrying the reason, the check results and the halt it + // supersedes, then confirms by read-back exactly as markReorgHalted does. + // Returns { cleared, alreadyClear }. Never deletes the halt row. + async clearReorgHalt({ reason, checks = {}, forced = false } = {}){ + if (typeof reason !== 'string' || reason.trim().length < 8) + throw new Error('clearReorgHalt: a reason of at least 8 characters is required; it is recorded with the clear') + const state = await this.readReorgHaltState() + if (!state.halted) return { cleared: false, alreadyClear: true } + const written = await this.insertEvent('REORG_HALT_CLEARED', { + reason: reason.trim(), + at: new Date().toISOString(), + forced: !!forced, + checks: checks, + cleared_halt_id: state.id, + cleared_halt_at: state.at, + cleared_halt_reason: state.reason + }) + if (written !== true) return { cleared: false, alreadyClear: false } + const after = await this.readReorgHaltState() + return { cleared: after.halted === false, alreadyClear: false } + } + // How many distinct block heights above the current tip have already been // rolled back and not yet re-synced. // @@ -2762,29 +2853,18 @@ class Database { // are null when the row exists but its payload is unreadable (an older marker, or // JSON written by a different revision), which must never turn a real halt into a // reported non-halt. + // + // Honours an operator clear: after clearReorgHalt the marker reads as not + // halted and carries `cleared_at` / `cleared_reason` instead, so the health + // surface can show that a halt WAS here and who cleared it. async getReorgHaltMarker(){ - const query = `SELECT time, data FROM events WHERE code = 'REORG_HALT' ORDER BY id DESC LIMIT 1;` - let connection = await this.getConnection() - const ownLease = (this.transactionConnection == null) - try { - const rows = await connection.query(query) - if (!Array.isArray(rows) || rows.length === 0) return { halted: false, at: null, reason: null } - const row = rows[0] - let payload = null - try { - payload = (typeof row.data === 'string') ? JSON.parse(row.data) : row.data - } catch (_) { - payload = null - } - return { - halted: true, - at: (payload && payload.at) ? payload.at : (row.time != null ? String(row.time) : null), - reason: (payload && payload.reason) ? payload.reason : null - } - } finally { - if (ownLease){ - await connection.release() - } + const state = await this.readReorgHaltState() + return { + halted: state.halted, + at: state.at, + reason: state.reason, + cleared_at: state.cleared_at, + cleared_reason: state.cleared_reason } } diff --git a/src/observability/logShipper.js b/src/observability/logShipper.js index dae9542..a36c827 100644 --- a/src/observability/logShipper.js +++ b/src/observability/logShipper.js @@ -84,7 +84,7 @@ const ENVELOPE_KEYS = new Set(['ts', 'level', 'service', 'msg', 'version']); // A bare token only where it cannot be confused with the next pair: anything // carrying whitespace, `=` or a quote is JSON-quoted so a reader can split the // tail on unquoted spaces. This is the half of the text format the watch -// collector's parser is written against (claude/scripts/xchain-watch.js). +// collector's parser is written against. function formatFieldValue(value) { if (value === null) return 'null'; if (typeof value === 'string') { diff --git a/test/unit/decoderHaltDiagnostics.test.js b/test/unit/decoderHaltDiagnostics.test.js index 385bf1e..9cc9ba3 100644 --- a/test/unit/decoderHaltDiagnostics.test.js +++ b/test/unit/decoderHaltDiagnostics.test.js @@ -61,11 +61,14 @@ function makeDecoder() { ) } -// A decoder holding blocks far above the node's tip, so verifyReorg takes its -// above-tip delete branch and rolls back one block per pass until the -// safe-depth ceiling aborts. The db carries only what that walk reads, so the -// halt these cases assert on is the real one and not a stubbed shortcut. -const NODE_TIP = 100 +// A decoder one block above the node's tip whose every stored hash disagrees +// with the node, so verifyReorg deletes the above-tip block and then walks the +// hash-compare back one block per pass until the safe-depth ceiling aborts. (A +// gap the ceiling could not absorb is refused before the first delete and never +// reaches the halt; that is nodeCatchUpWait.test.js.) The db carries only what +// that walk reads, so the halt these cases assert on is the real one and not a +// stubbed shortcut. +const NODE_TIP = 299 function haltingDecoder(db) { const decoder = makeDecoder() @@ -75,7 +78,7 @@ function haltingDecoder(db) { getBlockByIndex: async (i) => (i < 0 ? null : { block_index: i, block_hash: 'aa'.repeat(32) }), deleteBlockByIndex: async () => { height -= 1; return true } }, db) - decoder.connector = { rpcErrors: 0 } + decoder.connector = { rpcErrors: 0, getBlockHash: async () => 'bb'.repeat(32) } return decoder } diff --git a/test/unit/nodeCatchUpWait.test.js b/test/unit/nodeCatchUpWait.test.js new file mode 100644 index 0000000..de1ced1 --- /dev/null +++ b/test/unit/nodeCatchUpWait.test.js @@ -0,0 +1,173 @@ +/********************************************************************* + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * A node tip below the stored tip is not always a rollback. + * + * Measured on an operator's fresh BTC mainnet node (2026-09-07): the node was + * still in initial block download at 962304 while the decoder held 964970. The + * tip-regression branch called it orphans, verifyReorg deleted 126 valid blocks + * to the safe-depth ceiling, wrote the durable REORG_HALT and the container + * crash-looped 279 times, over a reorg that never happened. Two guards close it: + * + * 1. the parse loop reads initialblockdownload from the reply it already holds + * and WAITS while it is true, instead of reconciling; + * 2. verifyReorg's above-tip branch, which knows its depth before the first + * delete, refuses up front when that depth cannot fit the window, with + * nothing deleted and no durable halt (nothing was lost). + */ + +'use strict' + +const assert = require('assert') +const fs = require('fs') +const path = require('path') +const XChainDecoder = require('../../src/XChainDecoder') +const { nodeStillCatchingUp, DISPENSER_EXPIRE_SAFE_DEPTH } = XChainDecoder + +const SAFE_DEPTH = DISPENSER_EXPIRE_SAFE_DEPTH + +function makeDecoder() { + const decoder = new XChainDecoder( + 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ) + decoder.startBlockIndex = 0 + decoder.sleep = async () => {} + return decoder +} + +// Stored tip at `top`; the node agrees with every stored hash, so once the walk +// reaches the node tip it ends. `priorDepth` is what a previous process already +// rolled back above the tip (the restart-durable count). +function decoderAbove(top, { priorDepth = 0 } = {}) { + const decoder = makeDecoder() + const deleted = [] + let marked = 0 + decoder.connector = { rpcErrors: 0, getBlockHash: async (h) => 'hash' + h } + decoder.db = { + getLastBlockIndex: async () => top, + getBlockByIndex: async (h) => (h < 0 ? null : { block_index: h, block_hash: 'hash' + h }), + deleteBlockByIndex: async (h) => { deleted.push(h); top = h - 1; return true }, + isReorgHalted: async () => false, + countReorgDeletesAboveTip: async () => priorDepth, + markReorgHalted: async () => { marked++; return true } + } + return { decoder, deleted, marked: () => marked } +} + +describe('nodeStillCatchingUp(): the IBD read off getblockchaininfo', function () { + it('is true only for a literal initialblockdownload=true', function () { + assert.strictEqual(nodeStillCatchingUp({ initialblockdownload: true }), true) + assert.strictEqual(nodeStillCatchingUp({ initialblockdownload: false }), false) + }) + + it('fails open on an absent, null or non-boolean field (older node, trimmed proxy)', function () { + assert.strictEqual(nodeStillCatchingUp({ blocks: 10 }), false) + assert.strictEqual(nodeStillCatchingUp({ initialblockdownload: null }), false) + assert.strictEqual(nodeStillCatchingUp({ initialblockdownload: 'true' }), false) + assert.strictEqual(nodeStillCatchingUp({ initialblockdownload: 1 }), false) + assert.strictEqual(nodeStillCatchingUp(null), false) + assert.strictEqual(nodeStillCatchingUp(undefined), false) + }) +}) + +describe('verifyReorg: an above-tip gap the window cannot absorb is refused before the first delete', function () { + + it('deletes nothing and writes no halt when the known depth alone exceeds the ceiling', async function () { + const { decoder, deleted, marked } = decoderAbove(300) + + await assert.rejects(() => decoder.verifyReorg(300 - SAFE_DEPTH - 1), (err) => { + assert.strictEqual(err.tipBelowStoredTip, true, 'tagged so the parse loop can wait on it') + assert.match(err.message, /127 blocks below the stored tip/) + assert.match(err.message, /no REORG_HALT marker was written/) + assert.match(err.message, /needs no resync/) + return true + }) + + assert.strictEqual(deleted.length, 0, 'the whole point: not one block before the refusal') + assert.strictEqual(marked(), 0, 'no durable halt: nothing was rolled back') + assert.strictEqual(decoder.getReorgHaltStatus().halted, false) + }) + + it('a gap of exactly the ceiling still reconciles (the ceiling is a budget, not a fence)', async function () { + const { decoder, deleted } = decoderAbove(300) + + assert.strictEqual(await decoder.verifyReorg(300 - SAFE_DEPTH), true) + assert.strictEqual(deleted.length, SAFE_DEPTH) + assert.strictEqual(decoder.getReorgHaltStatus().halted, false) + }) + + it('counts what a previous process already rolled back toward the refusal', async function () { + // 10 above the tip would fit a fresh window; it does not fit the 120 a + // killed process already spent, and a restart must not delete the 6 that + // remain just to abort on the 7th. + const { decoder, deleted } = decoderAbove(300, { priorDepth: SAFE_DEPTH - 6 }) + + await assert.rejects(() => decoder.verifyReorg(290), (err) => { + assert.strictEqual(err.tipBelowStoredTip, true) + assert.match(err.message, /with 120 block\(s\) already rolled back/) + return true + }) + assert.strictEqual(deleted.length, 0) + }) + + it('a prior depth already AT the ceiling still takes the durable halt, not the refusal', async function () { + // Blocks past the window are already gone from this database: that is the + // halt's case, and the refusal must not soften it. + const { decoder, deleted, marked } = decoderAbove(300, { priorDepth: SAFE_DEPTH }) + + await assert.rejects(() => decoder.verifyReorg(299), /safe-depth/) + assert.strictEqual(deleted.length, 0) + assert.strictEqual(marked(), 1) + assert.strictEqual(decoder.getReorgHaltStatus().halted, true) + }) +}) + +describe('the parse loop waits on a node in initial block download instead of reconciling', function () { + const SRC = fs.readFileSync(path.join(__dirname, '..', '..', 'src', 'XChainDecoder.js'), 'utf8') + + // The branch under test needs a live node whose tip sits below the stored tip + // and a full start() loop to reach, so this is a source-level drift guard in + // the shape of chainIdentityGate.test.js: the IBD check has to sit between + // the tip-regression detection and the verifyReorg call, and it has to wait. + const branchStart = SRC.indexOf('is still behind the starting block') + const reconcile = SRC.indexOf('Reconciling orphan blocks...') + + it('the tip-regression branch exists in the order the guard relies on', function () { + assert.ok(branchStart > 0 && reconcile > branchStart) + }) + + it('reads initialblockdownload off the reply it already holds, before the reconcile', function () { + const between = SRC.slice(branchStart, reconcile) + assert.ok(/nodeStillCatchingUp\(lastBlockchainInfo\)/.test(between), + 'the IBD check must precede the orphan reconcile in the tip-regression branch') + }) + + it('waits (sleep + continue) rather than calling verifyReorg while IBD is true', function () { + const between = SRC.slice(branchStart, reconcile) + const at = between.indexOf('nodeStillCatchingUp(lastBlockchainInfo)') + const after = between.slice(at, at + 900) + assert.ok(/await this\.sleep\(\d+\)/.test(after), 'the IBD branch must sleep') + assert.ok(/continue/.test(after), 'the IBD branch must re-poll, not fall through') + assert.ok(!/verifyReorg/.test(after), 'the IBD branch must never reconcile') + }) + + it('a pre-delete refusal from verifyReorg is waited on, not thrown out of the loop', function () { + const after = SRC.slice(reconcile, reconcile + 2200) + assert.ok(/err\.tipBelowStoredTip/.test(after), + 'the tip-regression call site must recognise the tagged refusal') + const at = after.indexOf('err.tipBelowStoredTip') + const handler = after.slice(at, at + 500) + assert.ok(/await this\.sleep\(\d+\)/.test(handler) && /continue/.test(handler), + 'the refusal handler must sleep and re-poll the tip') + }) +}) diff --git a/test/unit/nodeCatchingUpStatus.test.js b/test/unit/nodeCatchingUpStatus.test.js new file mode 100644 index 0000000..7863ff2 --- /dev/null +++ b/test/unit/nodeCatchingUpStatus.test.js @@ -0,0 +1,264 @@ +/********************************************************************* + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * The IBD wait, made visible. + * + * nodeCatchUpWait.test.js pinned the WAIT: a node reporting + * initialblockdownload=true with its tip below ours is waited on, never + * reconciled. That wait is silent, and silence is what it looks like from + * outside: the decoder's height stops moving, every health surface still reads + * green, and nothing on the wire says why. An operator watching `xchain-node ps` + * sees a decoder that has stopped, indistinguishable from a wedge. + * + * These pin the surface: `this.nodeCatchingUp` is null unless the loop is inside + * that wait, carries the two heights and the instant the wait began while it is, + * clears when the wait ends, and rides every health payload that already carries + * reorg_halted. + */ + +'use strict' + +const assert = require('assert') +const fs = require('fs') +const http = require('http') +const path = require('path') +const express = require('express') + +const XChainDecoder = require('../../src/XChainDecoder') +const { registerLiveRoute } = require('../../src/api') + +const STORED_TIP = 100 + +// A decoder whose start() reaches the parse loop against mocks only (the shape +// parseLoopQuarantine.test.js drives), holding STORED_TIP while the node answers +// from `infoQueue`, one entry per poll. The last entry repeats if the loop outlives +// the queue. `onPoll` runs after each answer is handed out, which is how a test +// stops the loop: stopFlag is read at the TOP of the next iteration, so setting it +// here lets the current poll run to completion first. +function buildDecoder(infoQueue, onPoll = () => {}){ + const decoder = new XChainDecoder( + 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ) + decoder.startBlockIndex = 0 + + const waits = [] + let polls = 0 + + // Every wait-branch iteration ends in sleep(), so a snapshot taken here is the + // state the health surfaces would have published on that poll. + decoder.sleep = async () => { + if (decoder.nodeCatchingUp) waits.push(Object.assign({}, decoder.nodeCatchingUp)) + } + + decoder.connector = { + rpcErrors: 0, + getBlockchainInfo: async () => { + const info = infoQueue[Math.min(polls, infoQueue.length - 1)] + polls++ + onPoll(polls, decoder) + return Object.assign({ verificationprogress: 1 }, info) + }, + getBlockHash: async () => 'aabbccdd', + getBlock: async () => '' + } + + decoder.db = { + createDatabase: async () => true, + verifyDatabase: async () => true, + verifyTables: async () => true, + runMigrations: async () => ({ applied: [], pending: [] }), + getLastBlockIndex: async () => STORED_TIP, + getLastTxIndex: async () => 0, + endTransaction: async () => {}, + ping: async () => true + } + + return { decoder, waits, pollCount: () => polls } +} + +// A clock the test advances by hand, so "the timestamp did not move" is a real +// claim: with the wall clock, two polls of a loop whose sleep is a no-op can land +// in the same millisecond and a re-derived `since` would look stable by accident. +function withFrozenClock(run){ + const RealDate = global.Date + let now = RealDate.UTC(2026, 8, 8, 12, 0, 0) + class FakeDate extends RealDate { + constructor(...args){ args.length ? super(...args) : super(now) } + static now(){ return now } + } + global.Date = FakeDate + const tick = (ms) => { now += ms } + return Promise.resolve(run(tick)).finally(() => { global.Date = RealDate }) +} + +describe('the IBD wait is published as node_catching_up', function () { + this.timeout(0) + + it('is null on a fresh decoder, so no surface has to invent the not-waiting state', function () { + const decoder = new XChainDecoder( + 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ) + assert.strictEqual(decoder.nodeCatchingUp, null) + }) + + it('carries both heights and the instant the wait began while the node is in IBD', async function () { + await withFrozenClock(async (tick) => { + // Two IBD polls with the node advancing between them, then stop. + const { decoder, waits } = buildDecoder( + [ + { blocks: 50, initialblockdownload: true }, + { blocks: 60, initialblockdownload: true } + ], + (polls, d) => { tick(5000); if (polls >= 2) d.stopFlag = true } + ) + + await decoder.start() + + assert.strictEqual(waits.length, 2, 'both polls must have waited, not reconciled') + assert.deepStrictEqual(Object.keys(waits[0]).sort(), + ['node_height', 'since', 'stored_height']) + + assert.strictEqual(waits[0].node_height, 50) + assert.strictEqual(waits[0].stored_height, STORED_TIP) + assert.strictEqual(waits[1].node_height, 60, 'the node height is re-read every poll') + assert.strictEqual(waits[1].stored_height, STORED_TIP) + + assert.strictEqual(waits[0].since, waits[1].since, + 'since names when THIS wait began; a per-poll rewrite makes every wait look brand new') + assert.strictEqual(waits[0].since, new Date(waits[0].since).toISOString(), + 'an ISO instant, not a locale string') + }) + }) + + it('clears when the node leaves initial block download, on the same transition as the log', async function () { + const { decoder, waits } = buildDecoder( + [ + { blocks: 50, initialblockdownload: true }, + { blocks: 50, initialblockdownload: false } + ], + (polls, d) => { if (polls >= 2) d.stopFlag = true } + ) + // Past the transition the gap is a rollback again; the reconcile itself is + // nodeCatchUpWait.test.js's subject, not this one's. + let reconciled = 0 + decoder.verifyReorg = async () => { reconciled++; return true } + + await decoder.start() + + assert.strictEqual(waits.length, 1, 'only the IBD poll waits') + assert.strictEqual(reconciled, 1, 'control: leaving IBD hands the gap back to the reorg path') + assert.strictEqual(decoder.nodeCatchingUp, null, 'the wait must not stay latched on the surfaces') + }) + + it('clears above the tip-regression branch, which a caught-up node never enters again', function () { + const SRC = fs.readFileSync(path.join(__dirname, '..', '..', 'src', 'XChainDecoder.js'), 'utf8') + const branch = SRC.indexOf('if (lastProcessedBlockIndex > this.blockchainInfoLastBlock){') + assert.ok(branch > 0, 'the tip-regression branch must still be there to clear above') + + const before = SRC.slice(Math.max(0, branch - 800), branch) + assert.ok(/if \(this\.nodeCatchingUp && lastProcessedBlockIndex <= this\.blockchainInfoLastBlock\)\{\s*\n\s*this\.nodeCatchingUp = null/.test(before), + 'the wait must be cleared BEFORE the branch: the usual exit is the node reaching our height, ' + + 'which stops entering the branch at all and would strand a finished wait on ps forever') + assert.ok(!/nodeCatchingUpProblem/.test(before), + 'the log latch belongs to the branch below and must not be cleared here') + }) + + it('clears when the node overtakes the stored tip without a below-tip poll in between', async function () { + // The other exit, and the one the leaving-IBD transition structurally cannot + // see: it only fires while the node tip is STILL below ours. + const { decoder } = buildDecoder( + [ + { blocks: 50, initialblockdownload: true }, + { blocks: 150, initialblockdownload: false } + ], + (polls, d) => { if (polls >= 2) d.stopFlag = true } + ) + // Stop at the block fetch: the exit under test is upstream of the parse path, + // and a fetch failure is the loop's own sleep-and-retry, not an escape. + decoder.fetchBlockHex = async () => { throw new Error('test: stop before parsing') } + + await decoder.start() + + assert.strictEqual(decoder.nodeCatchingUp, null) + }) +}) + +describe('node_catching_up rides the health payloads', function () { + const API = fs.readFileSync(path.join(__dirname, '..', '..', 'src', 'api.js'), 'utf8') + + function liveApp(decoder, running = true){ + const app = express() + registerLiveRoute(app, decoder, () => running) + return app + } + + function getLive(app){ + return new Promise((resolve, reject) => { + const server = app.listen(0, () => { + http.get({ port: server.address().port, path: '/live' }, (res) => { + let body = '' + res.on('data', (c) => { body += c }) + res.on('end', () => { server.close(); resolve({ status: res.statusCode, body: JSON.parse(body) }) }) + }).on('error', (e) => { server.close(); reject(e) }) + }) + }) + } + + function probeDecoder(){ + const decoder = new XChainDecoder( + 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ) + decoder.lastProcessedBlockIndex = STORED_TIP + decoder.blockchainInfoLastBlock = 50 + decoder.blockchainInfoLastRefreshAt = Date.now() + decoder.lastAdvanceAt = Date.now() + decoder.lastPollAt = Date.now() + decoder.db = { ping: async () => true } + decoder.connector = { rpcErrors: 0 } + return decoder + } + + it('/live publishes the wait verbatim (the real registrar, not a copy of it)', async function () { + const decoder = probeDecoder() + decoder.nodeCatchingUp = { node_height: 50, stored_height: STORED_TIP, since: '2026-09-08T12:00:00.000Z' } + const res = await getLive(liveApp(decoder)) + assert.deepStrictEqual(res.body.node_catching_up, decoder.nodeCatchingUp) + }) + + it('/live publishes null when no wait is running, never an omitted key', async function () { + const res = await getLive(liveApp(probeDecoder())) + assert.ok('node_catching_up' in res.body, 'absent reads as "this build cannot tell you", not "not waiting"') + assert.strictEqual(res.body.node_catching_up, null) + }) + + // /status and the JSON-RPC health method are built inside startApi(), which binds a + // port and a live decoder, so these two are pinned at source level: the field must + // ship beside reorg_halted on every payload that carries it, and must read through a + // null fallback so a payload built without a decoder instance cannot throw. + it('every payload carrying reorg_halted also carries node_catching_up', function () { + const sites = [] + for (let at = API.indexOf('reorg_halted:'); at !== -1; at = API.indexOf('reorg_halted:', at + 1)) sites.push(at) + assert.strictEqual(sites.length, 3, 'three payloads carry reorg_halted: /live, rpc health, /status') + + for (const at of sites){ + const block = API.slice(at, at + 600) + assert.ok(/node_catching_up:/.test(block), + 'a reorg_halted payload at offset ' + at + ' ships without node_catching_up') + } + }) + + it('reads the field fail-soft, so an absent decoder cannot throw a payload', function () { + const reads = API.match(/node_catching_up:\s*\(decoder && decoder\.nodeCatchingUp\) \|\| null/g) || [] + assert.strictEqual(reads.length, 3, 'each site must guard the instance and default to null') + }) +}) diff --git a/test/unit/reorgDepthAcrossRestart.test.js b/test/unit/reorgDepthAcrossRestart.test.js index 0a89734..fd1093c 100644 --- a/test/unit/reorgDepthAcrossRestart.test.js +++ b/test/unit/reorgDepthAcrossRestart.test.js @@ -29,7 +29,11 @@ const XChainDecoder = require('../../src/XChainDecoder') const Database = require('../../src/db.js') const SAFE_DEPTH = 126 -const NODE_TIP = 100 +// One block above the node tip, so the walk's known above-tip depth never trips +// the pre-delete refusal (a gap the ceiling could not absorb is refused before +// the first delete and is its own test file); the fork below the tip is what +// these cases spend the window on. +const NODE_TIP = 299 function makeDecoder() { return new XChainDecoder( @@ -37,9 +41,10 @@ function makeDecoder() { ) } -// A decoder holding blocks far above the node tip, so verifyReorg takes its -// above-tip delete branch and rolls back one block per pass. `db` overrides let -// each case state only the restart evidence it is about. +// A decoder one block above the node tip whose every stored hash disagrees with +// the node, so verifyReorg deletes the above-tip block and then walks the +// hash-compare back one block per pass until the ceiling fires. `db` overrides +// let each case state only the restart evidence it is about. function restartedDecoder(db) { const decoder = makeDecoder() let height = 300 @@ -53,7 +58,7 @@ function restartedDecoder(db) { isReorgHalted: async () => false, markReorgHalted: async () => true }, db) - decoder.connector = { rpcErrors: 0 } + decoder.connector = { rpcErrors: 0, getBlockHash: async () => 'bb'.repeat(32) } // The seed read retries with a 3s sleep; no test may pay for that in wall time. decoder.sleep = async () => {} return { decoder, deleted } diff --git a/test/unit/reorgHaltClear.test.js b/test/unit/reorgHaltClear.test.js new file mode 100644 index 0000000..13639d1 --- /dev/null +++ b/test/unit/reorgHaltClear.test.js @@ -0,0 +1,201 @@ +/********************************************************************* + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * The audited REORG_HALT clear. + * + * A halt marker used to be cleared only by rebuilding the schema, so a database + * nothing had been purged from still owed a full resync. The clear writes a + * REORG_HALT_CLEARED row (reason, checks, the halt it supersedes) and the newest + * of the two codes decides; the halt row is never deleted. + */ + +'use strict' + +const assert = require('assert') +const sinon = require('sinon') +const Database = require('../../src/db.js') +const { run, parseArgs, EXIT } = require('../../src/clear-reorg-halt.js') + +function dbAnswering(handler) { + const db = new Database('127.0.0.1', 3306, 'xchain_btc_mainnet', 'u', 'p') + const query = sinon.stub().callsFake(async (sql, params) => handler(sql, params)) + db.pool = { getConnection: sinon.stub().resolves({ query, release: sinon.stub().resolves() }) } + return { db, query } +} + +const halt = (id) => ({ id, time: '2026-09-07 06:29:07', code: 'REORG_HALT', data: JSON.stringify({ reason: 'safe-depth', at: '2026-09-07T06:29:07Z' }) }) +const cleared = (id) => ({ id, time: '2026-09-08 10:00:00', code: 'REORG_HALT_CLEARED', data: JSON.stringify({ reason: 'zero dispensers', at: '2026-09-08T10:00:00Z' }) }) + +describe('Database: the newest REORG_HALT / REORG_HALT_CLEARED row decides', function () { + afterEach(() => sinon.restore()) + + it('a halt with no clear is live', async function () { + const { db } = dbAnswering(() => [halt(7)]) + assert.strictEqual(await db.isReorgHalted(), true) + const m = await db.getReorgHaltMarker() + assert.strictEqual(m.halted, true) + assert.strictEqual(m.reason, 'safe-depth') + assert.strictEqual(m.cleared_at, null) + }) + + it('a clear newer than the halt reads as not halted, and says when and why it was cleared', async function () { + const { db } = dbAnswering(() => [cleared(9)]) + assert.strictEqual(await db.isReorgHalted(), false) + const m = await db.getReorgHaltMarker() + assert.deepStrictEqual(m, { halted: false, at: null, reason: null, cleared_at: '2026-09-08T10:00:00Z', cleared_reason: 'zero dispensers' }) + }) + + it('no row at all is not halted', async function () { + const { db } = dbAnswering(() => []) + assert.strictEqual(await db.isReorgHalted(), false) + assert.strictEqual((await db.getReorgHaltMarker()).halted, false) + }) + + it('asks for the newest of BOTH codes in one query', async function () { + const { db, query } = dbAnswering(() => []) + await db.isReorgHalted() + const sql = String(query.firstCall.args[0]) + assert.match(sql, /code IN \('REORG_HALT', 'REORG_HALT_CLEARED'\)/) + assert.match(sql, /ORDER BY id DESC LIMIT 1/) + }) + + it('a halt row with an unreadable code or payload still counts as live (fail-closed)', async function () { + const { db } = dbAnswering(() => [{ id: 3, time: 't', code: undefined, data: '{not json' }]) + assert.strictEqual(await db.isReorgHalted(), true) + }) + + it('clearReorgHalt writes a REORG_HALT_CLEARED row that supersedes the halt and confirms by read-back', async function () { + let state = [halt(7)] + const inserted = [] + const { db } = dbAnswering((sql, params) => { + if (/INSERT INTO events/.test(sql)) { inserted.push(params); state = [cleared(8)]; return { affectedRows: 1 } } + return state + }) + const res = await db.clearReorgHalt({ reason: 'mainnet decoder, zero dispensers, range re-synced', checks: { dispensers: 0 }, forced: false }) + assert.deepStrictEqual(res, { cleared: true, alreadyClear: false }) + assert.strictEqual(inserted.length, 1) + assert.strictEqual(inserted[0][1], 'REORG_HALT_CLEARED') + const payload = JSON.parse(inserted[0][2]) + assert.strictEqual(payload.reason, 'mainnet decoder, zero dispensers, range re-synced') + assert.strictEqual(payload.cleared_halt_id, 7) + assert.strictEqual(payload.cleared_halt_reason, 'safe-depth') + assert.deepStrictEqual(payload.checks, { dispensers: 0 }) + assert.strictEqual(payload.forced, false) + }) + + it('clearReorgHalt is a no-op on a database that is not halted', async function () { + const { db, query } = dbAnswering(() => []) + assert.deepStrictEqual(await db.clearReorgHalt({ reason: 'long enough reason' }), { cleared: false, alreadyClear: true }) + assert.ok(!query.getCalls().some(c => /INSERT/.test(String(c.args[0]))), 'nothing written') + }) + + it('clearReorgHalt refuses a missing or trivial reason before touching the database', async function () { + const { db, query } = dbAnswering(() => [halt(7)]) + await assert.rejects(() => db.clearReorgHalt({ reason: 'ok' }), /reason of at least 8 characters/) + await assert.rejects(() => db.clearReorgHalt({}), /reason of at least 8 characters/) + assert.strictEqual(query.callCount, 0) + }) + + it('clearReorgHalt reports not-cleared when the write does not land', async function () { + const { db } = dbAnswering((sql) => { + if (/INSERT INTO events/.test(sql)) throw Object.assign(new Error('disk full'), { errno: 1 }) + return [halt(7)] + }) + assert.deepStrictEqual(await db.clearReorgHalt({ reason: 'long enough reason' }), { cleared: false, alreadyClear: false }) + }) + + it('a later halt after a clear is live again', async function () { + const { db } = dbAnswering(() => [halt(12)]) + assert.strictEqual(await db.isReorgHalted(), true) + }) +}) + +describe('clear-reorg-halt CLI', function () { + function fakeDb({ halted = true, deletesAboveTip = 0, dispensers = 0, dispenserTxs = false, clearResult = { cleared: true, alreadyClear: false } } = {}) { + const calls = { clear: [] } + const db = { + getReorgHaltMarker: async () => (halted ? { halted: true, at: '2026-09-07T06:29:07Z', reason: 'safe-depth', cleared_at: null, cleared_reason: null } + : { halted: false, at: null, reason: null, cleared_at: '2026-09-08T10:00:00Z', cleared_reason: 'earlier clear' }), + countReorgDeletesAboveTip: async () => deletesAboveTip, + countDispensers: async () => dispensers, + hasDispenserTransactions: async () => dispenserTxs, + clearReorgHalt: async (opts) => { calls.clear.push(opts); return clearResult } + } + return { db, calls } + } + const quiet = { log: () => {}, error: () => {} } + const REASON = 'BTC mainnet decoder, no dispensers exist yet, block range intact' + + it('parses --reason, --force and --dry-run', function () { + assert.deepStrictEqual(parseArgs(['--reason', 'x y z', '--force', '--dry-run']), + { reason: 'x y z', force: true, dryRun: true, help: false, bad: null }) + assert.strictEqual(parseArgs(['--reason=inline']).reason, 'inline') + assert.match(parseArgs(['--reason']).bad, /requires a text argument/) + assert.match(parseArgs(['--bogus']).bad, /unknown argument/) + }) + + it('refuses without a substantive reason and writes nothing', async function () { + const { db, calls } = fakeDb() + assert.strictEqual(await run({ db, argv: [], ...quiet }), EXIT.USAGE) + assert.strictEqual(await run({ db, argv: ['--reason', 'short'], ...quiet }), EXIT.USAGE) + assert.strictEqual(calls.clear.length, 0) + }) + + it('clears a clean database and records the checks', async function () { + const { db, calls } = fakeDb() + const lines = [] + assert.strictEqual(await run({ db, argv: ['--reason', REASON], log: (l) => lines.push(l), error: quiet.error }), EXIT.OK) + assert.strictEqual(calls.clear.length, 1) + assert.strictEqual(calls.clear[0].reason, REASON) + assert.strictEqual(calls.clear[0].forced, false) + assert.deepStrictEqual(calls.clear[0].checks, { deletes_above_tip: 0, dispensers: 0, dispenser_transactions: false }) + assert.ok(lines.some(l => /cleared\./.test(l))) + }) + + it('is a no-op when no halt is live', async function () { + const { db, calls } = fakeDb({ halted: false }) + const lines = [] + assert.strictEqual(await run({ db, argv: ['--reason', REASON], log: (l) => lines.push(l), error: quiet.error }), EXIT.OK) + assert.strictEqual(calls.clear.length, 0) + assert.ok(lines.some(l => /no live REORG_HALT marker/.test(l) && /earlier clear/.test(l))) + }) + + it('refuses, and cannot be forced, while rolled-back blocks are still missing above the tip', async function () { + const { db, calls } = fakeDb({ deletesAboveTip: 5 }) + assert.strictEqual(await run({ db, argv: ['--reason', REASON, '--force'], ...quiet }), EXIT.NOT_RESYNCED) + assert.strictEqual(calls.clear.length, 0) + }) + + it('refuses a database that has held dispenser state unless forced, and records the force', async function () { + const { db, calls } = fakeDb({ dispensers: 3 }) + assert.strictEqual(await run({ db, argv: ['--reason', REASON], ...quiet }), EXIT.DISPENSER_STATE) + assert.strictEqual(calls.clear.length, 0) + + const forced = fakeDb({ dispenserTxs: true }) + assert.strictEqual(await run({ db: forced.db, argv: ['--reason', REASON, '--force'], ...quiet }), EXIT.OK) + assert.strictEqual(forced.calls.clear[0].forced, true) + }) + + it('--dry-run reports the verdict and writes nothing', async function () { + const { db, calls } = fakeDb() + const lines = [] + assert.strictEqual(await run({ db, argv: ['--reason', REASON, '--dry-run'], log: (l) => lines.push(l), error: quiet.error }), EXIT.OK) + assert.strictEqual(calls.clear.length, 0) + assert.ok(lines.some(l => /dry run/.test(l))) + }) + + it('reports failure when the clear row does not land', async function () { + const { db } = fakeDb({ clearResult: { cleared: false, alreadyClear: false } }) + assert.strictEqual(await run({ db, argv: ['--reason', REASON], ...quiet }), EXIT.FAILED) + }) +}) diff --git a/test/unit/reorgHaltSurface.test.js b/test/unit/reorgHaltSurface.test.js index ba21e51..6b64396 100644 --- a/test/unit/reorgHaltSurface.test.js +++ b/test/unit/reorgHaltSurface.test.js @@ -193,7 +193,7 @@ describe('Database.getReorgHaltMarker', function () { it('returns halted:false when no marker row exists', async function () { const { db, wasReleased } = stubDb([]) const marker = await db.getReorgHaltMarker() - assert.deepStrictEqual(marker, { halted: false, at: null, reason: null }) + assert.deepStrictEqual(marker, { halted: false, at: null, reason: null, cleared_at: null, cleared_reason: null }) assert.ok(wasReleased(), 'the pooled connection must be released') }) diff --git a/test/unit/verifyReorgRetry.test.js b/test/unit/verifyReorgRetry.test.js index 56c8212..8eb1401 100644 --- a/test/unit/verifyReorgRetry.test.js +++ b/test/unit/verifyReorgRetry.test.js @@ -140,9 +140,12 @@ describe('XChainDecoder.verifyReorg depth guard', function () { 'must stop deleting exactly at DISPENSER_EXPIRE_SAFE_DEPTH blocks') }) - it('also guards the above-tip orphan branch', async function () { - // Blocks stored above the node tip are deleted via a separate branch; a node - // rollback deeper than the window must trip the same fail-closed abort. + it('also guards the above-tip orphan branch, and there it refuses BEFORE the first delete', async function () { + // Blocks stored above the node tip are deleted via a separate branch. That + // branch knows its depth up front, so a node tip deeper below us than the + // window must not spend the window finding out: it refuses with nothing + // deleted and no durable halt (nothing was lost). The full contract is in + // nodeCatchUpWait.test.js. const decoder = new XChainDecoder( 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null ) @@ -159,7 +162,8 @@ describe('XChainDecoder.verifyReorg depth guard', function () { } // Node tip far below the stored tip: every stored block above it is an orphan. await assert.rejects(() => decoder.verifyReorg(10000 - SAFE_DEPTH - 50), /dispenser safe-depth window/) - assert.strictEqual(deleted.length, SAFE_DEPTH) + assert.strictEqual(deleted.length, 0) + assert.strictEqual(decoder.getReorgHaltStatus().halted, false) }) })