diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8c2469..5af882e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,16 @@ on: jobs: ci: uses: XChain-Platform/.github/.github/workflows/ci-reusable.yml@6f4d39ae85787fc31e90a31588d87610a2c33103 # pin: XChain-Platform/.github @ master 2026-08-14; bump deliberately + with: + # On a release or hotfix PR, check the siblings out at the PR's own + # branch so a release-branch-only change to a file a cross-repo guard + # reads is tested against the train, not against develop. The shared + # workflow otherwise pins siblings at develop for any pull request, + # which is what let this repo's release PRs go red on commits that + # touch neither the guard nor the file it reads. Empty everywhere + # else, which keeps the existing behaviour. Same expression as + # xchain-indexer and xchain-explorer. + siblings-ref: ${{ (startsWith(github.head_ref, 'release/') || startsWith(github.head_ref, 'hotfix/')) && github.head_ref || '' }} # Cross-repo drift guards: this repo vendors the canonical coin registry from # xchain-hub, and per-repo CI cannot see sibling repos, so a drifted vendored @@ -109,6 +119,12 @@ jobs: # Same roster (.ci-siblings) and same layout the shared workflow uses. - name: Check out declared sibling repositories uses: XChain-Platform/.github/actions/checkout-siblings@master # one definition for every call site; see the action for why this tracks master + # The ratchet re-runs the unit suite, so it needs the same release/hotfix + # awareness as the gate's own siblings-ref above, or it measures a suite + # whose cross-repo guards disagree with the gate for a reason that exists + # nowhere but in CI. + with: + ref: ${{ (startsWith(github.head_ref, 'release/') || startsWith(github.head_ref, 'hotfix/')) && github.head_ref || '' }} - name: Use Node.js 22 uses: actions/setup-node@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7727eac..2fa49a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +## [0.17.0] - 2026-09-10 + +### Added +- The health and status payloads carry `node_last_ok_at` and `node_unreachable`, so a coin node that has never answered is visible. + +### Changed +- The vendored mainnet activation twins are armed at genesis under the 2026-09-09 ruling. + +### Fixed +- `docker stop` now ends in a clean exit: the SIGTERM drain breaks the parse loop at a block boundary, closes the API listener and both database pools and exits 0 under a `SHUTDOWN_TIMEOUT_MS` hard-exit bound, where before the process parked until docker's SIGKILL. +- Nodes that applied the original `2026-05-28-unique-index-tables.sql` are healed instead of logging a checksum mismatch at every startup. + ## [0.16.0] - 2026-09-08 ### Added diff --git a/README.md b/README.md index 9a92047..aa586f0 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

Version - Tests + Tests Node License

@@ -113,7 +113,7 @@ neither source sets one, so these defaults hold on an unconfigured box: | `npm run migrate` | Apply pending database migrations (auto + manual; `--file ` scopes to specific migration(s)) | | `npm run ci` | The full no-external-services gate: unit, security, smoke, regression, chaos, and a 100-iteration fuzz pass (about a minute) | | `npm run test:smoke` | Smoke tests (58 tests, no external services) | -| `npm run test:unit` | Unit tests (1,549 tests, no external services) | +| `npm run test:unit` | Unit tests (1,593 tests, no external services) | | `npm run test:security` | Security tests (83 tests, no external services) | | `npm run test:integration` | Integration tests (30 tests; brings up its own throwaway regtest node and MariaDB, requires Docker) | | `npm run test:e2e` | End-to-end tests (72 tests; brings up its own throwaway regtest node and MariaDB on separate ports, requires Docker) | diff --git a/package-lock.json b/package-lock.json index 9a4b230..80f213e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "xchain-decoder", - "version": "0.16.0", + "version": "0.17.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "xchain-decoder", - "version": "0.16.0", + "version": "0.17.0", "license": "AGPL-3.0-or-later", "dependencies": { "axios": "^1.18.1", diff --git a/package.json b/package.json index 91646cf..d669edc 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "xchain-decoder", "description": "xchain-decoder decodes XChain platform transactions from a given blockchain and populates a database with the decoded data.", - "version": "0.16.0", + "version": "0.17.0", "license": "AGPL-3.0-or-later", "repository": { "type": "git", diff --git a/src/BlockchainConnector.js b/src/BlockchainConnector.js index 877ef6b..604be7a 100644 --- a/src/BlockchainConnector.js +++ b/src/BlockchainConnector.js @@ -286,12 +286,48 @@ function normalizeEndpoint(entry, defaultPort) { return protocol + match[2] + ':' + port } +// Reduce the three timestamps the connector records into the two fields every health +// surface publishes. Pure and exported so the rule lives in one place: a surface that +// re-derived "is the node reachable" from a counter would disagree with this one. +// +// Unreachable means the LATEST attempt failed: either nothing has ever succeeded, or +// the last failure is newer than the last success. `since` dates the outage from the +// last success when there was one, and from connector construction when there was +// never one, which is the case the defect report describes: a decoder whose node +// answered nothing in five and a half days while every surface read green. +// +// All three inputs are ms epoch, 0 meaning "never". +function nodeReachabilityFrom(startedAt, lastNodeOkAt, lastNodeFailAt, now = Date.now()) { + const lastOkIso = lastNodeOkAt > 0 ? new Date(lastNodeOkAt).toISOString() : null + const failing = lastNodeFailAt > 0 && (lastNodeOkAt === 0 || lastNodeFailAt > lastNodeOkAt) + if (!failing) return { node_last_ok_at: lastOkIso, node_unreachable: null } + const sinceMs = lastNodeOkAt > 0 ? lastNodeOkAt : startedAt + return { + node_last_ok_at: lastOkIso, + node_unreachable: { + since: new Date(sinceMs).toISOString(), + last_ok_at: lastOkIso, + // Floor, and clamped at 0: a health probe racing the recorded instant + // must never publish a negative age. + seconds: Math.max(0, Math.floor((now - sinceMs) / 1000)) + } + } +} + class BlockchainConnector { constructor(url, port, rpcUser, rpcPassword) { this.port = port this.rpcUser = rpcUser this.rpcPassword = rpcPassword this.rpcErrors = 0 + // Node reachability, recorded at the single POST choke point below so every + // RPC path through this class feeds it. Reported, never gated on: the healthy + // verdict deliberately ignores an upstream node outage (a restart cannot fix + // one, and gating re-opens the autoheal restart flap), which is exactly why the + // outage needs a surface of its own. + this.startedAt = Date.now() + this.lastNodeOkAt = 0 + this.lastNodeFailAt = 0 // RPC endpoint failover. A dead primary endpoint used to stall the // decoder forever, because the block loop retries RPC failures // indefinitely by design (skipping a block would corrupt the index). @@ -317,6 +353,12 @@ class BlockchainConnector { return this.endpoints[this.activeEndpointIndex] } + // Node reachability as the health surfaces publish it. Cheap and never throws, + // so a probe can call it per request. + nodeReachability(now = Date.now()) { + return nodeReachabilityFrom(this.startedAt, this.lastNodeOkAt, this.lastNodeFailAt, now) + } + // Single POST path for every RPC method: resets the consecutive-failure // counter on any answer from the node, and counts connection-level errors // toward failover before re-throwing for the caller's own retry handling. @@ -329,8 +371,16 @@ class BlockchainConnector { } }) this.connectionFailures = 0 + // The node answered. A JSON-RPC error carried in a 200 body (height out of + // range, tx not found) still resolves here and still counts as reached: + // this pair reports whether the node is ANSWERING, not whether the answer + // was the one the caller wanted. rpcErrors already counts the latter. + this.lastNodeOkAt = Date.now() return response } catch (error) { + // Timeouts (ECONNABORTED), socket/DNS faults and RPC errors delivered as + // HTTP 500 all land here, and all mean this attempt got no usable answer. + this.lastNodeFailAt = Date.now() if (error && error.response) { // An HTTP-level error (auth, queue-full 500, etc.) still proves // the endpoint is reachable; only unreachability drives failover. @@ -731,4 +781,6 @@ module.exports.encodeVarintHex = encodeVarintHex module.exports.stripAuxPowFromBlockHex = stripAuxPowFromBlockHex module.exports.skipAuxPow = skipAuxPow // Exported for the env-parsing regression test. -module.exports.envInt = envInt \ No newline at end of file +module.exports.envInt = envInt +// Exported so the reachability reducer can be tested without a connector or a node. +module.exports.nodeReachabilityFrom = nodeReachabilityFrom \ No newline at end of file diff --git a/src/api.js b/src/api.js index 6d781ec..890a4cc 100644 --- a/src/api.js +++ b/src/api.js @@ -38,6 +38,7 @@ const bodyParser = require('body-parser'); const helmet = require('helmet'); const cors = require('cors'); const rateLimit = require('express-rate-limit'); +const { createShutdown, createDecoderDrain } = require('./shutdown'); const XChainDecoder = require('./XChainDecoder'); const { resolveFeeDestination } = require('./feeDestination'); const jsonRouter = require('express-json-rpc-router') @@ -114,6 +115,26 @@ const AUX_POW = process.env.AUX_POW === 'true' || process.env.AUX_POW === '1' // outputs paying it to transaction_outputs so the indexer can validate native-coin fee payments. const FEE_DESTINATION = resolveFeeDestination(NETWORK, process.env.FEE_DESTINATION || null) +// Node reachability for the health payloads: `node_last_ok_at` (the last successful +// node RPC, null if there has never been one) and `node_unreachable` (null, or the +// outage with its age in seconds). A decoder whose node never answered a single RPC +// is otherwise indistinguishable from a healthy one on every surface an operator polls; +// these two fields are that difference, reported and never gating. +// +// Fail-soft: an absent connector, or one from a build/test stub predating the method, +// reports the unknown-but-not-failing pair rather than throwing inside a probe. +function nodeReachabilityFields(decoder){ + const connector = decoder && decoder.connector + if (!connector || typeof connector.nodeReachability !== 'function'){ + return { node_last_ok_at: null, node_unreachable: null } + } + try { + return connector.nodeReachability() + } catch (e) { + return { node_last_ok_at: null, node_unreachable: null } + } +} + // Express middleware that bounds JSON-RPC batch size. express-json-rpc-router runs // Promise.all over every element of a batch array, while the per-IP rate limiter counts // the whole batch as ONE request. Without a cap, a single ~100kb array of thousands of @@ -195,6 +216,10 @@ function registerLiveRoute(app, decoder, isDecoderRunning){ // { node_height, stored_height, since } while the parse loop is waiting out // a node in initial block download below our tip, null otherwise. node_catching_up: (decoder && decoder.nodeCatchingUp) || null, + // node_last_ok_at + node_unreachable. Same reporting-not-gating contract as + // node_height_stale below, and the only surface that separates "the node has + // never answered" from "the node is fine". + ...nodeReachabilityFields(decoder), // A frozen node tip, reported but deliberately NOT gating. isStalled() // returns false while the tip is stale on purpose: restarting the container // cannot fix an upstream node outage, and gating on it re-opens the @@ -224,7 +249,9 @@ async function startApi(){ const decoder = new XChainDecoder(NETWORK, DB_URL, DB_PORT, DECODER_DB_NAME, DECODER_DB_USER, DB_PASSWORD, NODE_URL, NODE_PORT, NODE_USER, NODE_PASSWORD, AUX_POW, FEE_DESTINATION); let decoderRunning = true let decoderError = null - decoder.start().then(() => { + // start() awaits the parse loop, so this promise SETTLES when the loop breaks: + // on a fatal error here, or on the stopFlag the drain sets at a block boundary. + const decoderExited = decoder.start().then(() => { // start() awaits the parse loop, so it RESOLVES only when the loop breaks: // the SIGTERM/stopFlag path, or any fall-through out of `while (true)`. // Without this, decoderRunning only ever went false on a REJECTION, so a @@ -263,18 +290,6 @@ async function startApi(){ process.exit(1) }) - // Graceful shutdown on process signals - const shutdown = () => { - console.log('Received shutdown signal, stopping decoder...') - // Flip BEFORE stop(): stop() only sets stopFlag, and the loop may take a - // whole iteration to notice. A drain must not answer /live with 200 in the - // window between the signal and the loop actually breaking. - decoderRunning = false - decoder.stop() - } - process.on('SIGTERM', shutdown) - process.on('SIGINT', shutdown) - // Crash visibility. Registered inside startApi(), not at module scope: several // unit suites require this module in-process under mocha to reach registerLiveRoute // and makeRpcBatchGuard, and mocha installs its own handlers. A module-scope @@ -415,6 +430,9 @@ async function startApi(){ // { node_height, stored_height, since } while the parse loop is waiting // out a node in initial block download below our tip, null otherwise. node_catching_up: (decoder && decoder.nodeCatchingUp) || null, + // node_last_ok_at + node_unreachable: whether the coin node is answering + // this decoder at all, and since when it stopped. Reported, not gated on. + ...nodeReachabilityFields(decoder), // Set once an operator cleared a halt (db.clearReorgHalt); null while a // halt is live or none was ever recorded. reorg_halt_cleared_at: reorgHalt.cleared_at || null, @@ -529,6 +547,9 @@ async function startApi(){ // { node_height, stored_height, since } while the parse loop is waiting out // a node in initial block download below our tip, null otherwise. node_catching_up: (decoder && decoder.nodeCatchingUp) || null, + // node_last_ok_at + node_unreachable: whether the coin node is answering + // this decoder at all, and since when it stopped. Reported, not gated on. + ...nodeReachabilityFields(decoder), // Ships beside the boolean, never without it. "Not halted" is only an answer // if something looked, and the probe is fail-soft: its state starts at // not-halted with checked_at null, so a decoder that has NEVER completed a @@ -555,9 +576,29 @@ async function startApi(){ app.use((req, res, next) => { if (req.body === undefined) req.body = {}; next(); }); app.use(jsonRouter({methods: jsonRpcController})) - app.listen(DECODER_API_PORT, () => { + const server = app.listen(DECODER_API_PORT, () => { console.log('API listening on port '+DECODER_API_PORT); }); + + // Graceful shutdown. node is PID 1 in the image, so `docker stop` delivers + // SIGTERM here. The earlier handler only set stopFlag: the loop broke, but + // this listener and the DB pool kept the process alive and nothing exited, + // so every stop ended in docker's SIGKILL. The drain is bounded by its own + // hard-exit timer (src/shutdown.js) because installing a handler removes + // node's default terminate. + const shutdown = createShutdown({ + drain: createDecoderDrain({ + decoder: decoder, + server: server, + loopSettled: decoderExited, + // Flip BEFORE stop(): stop() only sets stopFlag and the loop may take a + // whole iteration to notice. A drain must not answer /live with 200 in + // the window between the signal and the loop actually breaking. + onDraining: () => { decoderRunning = false } + }) + }) + process.on('SIGTERM', () => shutdown('SIGTERM')) + process.on('SIGINT', () => shutdown('SIGINT')) } // Auto-start only when run directly (node src/api.js), so the module can be required by @@ -567,4 +608,4 @@ if (require.main === module) startApi() // startApi is exported so the crash handlers it installs can be driven for real // rather than asserted against the source text; the require.main guard above // still keeps a plain require from opening a port or a DB connection. -module.exports = { makeRpcBatchGuard, registerLiveRoute, startApi, noteProbeFailure, _resetProbeLogState, _ageProbeLogState, PROBE_LOG_WINDOW_MS } \ No newline at end of file +module.exports = { makeRpcBatchGuard, registerLiveRoute, startApi, noteProbeFailure, nodeReachabilityFields, _resetProbeLogState, _ageProbeLogState, PROBE_LOG_WINDOW_MS } \ No newline at end of file diff --git a/src/batchSubCommandCapture.js b/src/batchSubCommandCapture.js index b469199..735b80a 100644 --- a/src/batchSubCommandCapture.js +++ b/src/batchSubCommandCapture.js @@ -453,10 +453,12 @@ function hasProvablyRejectedBatch(subCommands, aliases, consensusNetwork, blockT // // Its own vendored per-network instant, NOT the ordering argument the caps lean on. That // argument is specific to BATCH_ISSUANCE_LIMITS, whose instant the decoder's capture gate is -// required to sit at or after; the weighting flag has no such relationship and today it is -// the counter-example, with mainnet capture ARMED and the weighting instant still on the -// house sentinel. An absent or DISARMED (null) entry is inactive at every block time, which -// leaves today's over-capture in place rather than inventing a suppression rule. +// required to sit at or after; the weighting flag has the opposite relationship. Since the +// 2026-09-09 ruling armed it at mainnet genesis it sits BELOW capture there, and that is +// safe because the indexer applies the budget only inside its BATCH_ISSUANCE_LIMITS guard, +// which shares capture's instant: below it neither side weighs, and this mirror captures +// nothing to suppress. An absent or DISARMED (null) entry is inactive at every block time, +// which leaves over-capture in place rather than inventing a suppression rule. function isBatchCostWeightingActive(consensusNetwork, blockTime){ const activation = COST_WEIGHTING_ACTIVATION[consensusNetwork] if (typeof activation !== 'number') return false diff --git a/src/db.js b/src/db.js index 04731da..7febc10 100644 --- a/src/db.js +++ b/src/db.js @@ -117,6 +117,21 @@ class Database { } + // Drain support (src/shutdown.js): release a transaction connection still + // held, which the drain normally never sees because it waits for the parse + // loop to break at a block boundary, then end the pool so nothing keeps the + // event loop alive. Idempotent: a second call finds no pool and returns. + async close(){ + if(this.transactionConnection){ + try { await this.transactionConnection.release(); } catch(_){} + this.transactionConnection = null; + } + const pool = this.pool; + if(!pool) return; + this.pool = null; + await pool.end(); + } + // Seam over the driver: mariadb's createConnection export is // non-configurable, so tests stub this method instead of the module. _createConnection(connectionParams){ @@ -2905,9 +2920,10 @@ class Database { // DBs recorded whichever revision they applied first). Executable SQL is // byte-identical across every pinned revision (verified: strip `--` comment // lines and blank lines; the residue hashes identically from first commit to -// HEAD) for every entry EXCEPT the byte-order one at the bottom, which is -// justified by a measured data equivalence instead and carries that argument in -// full at its own entry rather than relying on this blanket sentence. +// HEAD) for every entry EXCEPT two, which are justified by a measured data +// equivalence instead and each carry that argument in full at its own entry +// rather than relying on this blanket sentence: the byte-order one at the +// bottom, and the 8151979 revision of the unique-index one. // Applied fleet-wide through code deploy: both the startup auto-run and // `node src/migrate.js` pass through this heal before the mismatch guard, so no // direct schema_migrations SQL is ever needed. Mirrors xchain-indexer/src/db.js. @@ -2980,12 +2996,42 @@ Database.MIGRATION_CHECKSUM_REBASELINES = { ], to: 'b03b41b6fcabef9c959851ede9b75cc9089cef7c015bdd69cfcea74ad5acea7a', // comment tidy (HEAD) }, - // Comment-only edit: the fleet recorded 50a5e83, which is the revision that ADDED the - // `@mempool_has_ids` guard, so the guarded UPDATEs are what actually ran. 7817e6c then - // added the license header. Stripped residue verified IDENTICAL between 50a5e83 and - // HEAD, so this entry meets the ordinary contract above. + // TWO revisions are pinned here and they are blessed for DIFFERENT reasons, so both are + // stated rather than filed together under the blanket sentence above. + // + // 50a5e83 (8845b9ad): the revision that ADDED the `@mempool_has_ids` guard, so the + // guarded UPDATEs are what actually ran. 7817e6c then added the license header. + // Stripped residue verified IDENTICAL between 50a5e83 and HEAD: ordinary contract. + // + // 8151979 (e1f7df79): the ORIGINAL shipped revision, applied by every node deployed in + // the 2026-06-10 .. 2026-07-10 window (one production BTC node among them, which is why its decoder + // logged the mismatch every startup). Its residue is NOT identical to HEAD's: 50a5e83 + // rewrote four mempool_transactions repoints from bare statements into + // `SET @s := IF(@mempool_has_ids, '', 'DO 0')` + PREPARE/EXECUTE. + // This is therefore a DATA equivalence, not a text one, and it is decided by the + // ledger row itself rather than assumed: + // + // - the recorded row EXISTS, so the file ran to completion on that database; + // - the 8151979 form references mempool_transactions.source_id / destination_id / + // tx_hash_id unguarded, so completion is only possible where those columns were + // present (otherwise MariaDB aborts the statement with errno 1054 and the runner + // records nothing); + // - columns present is exactly the branch HEAD's guard takes (@mempool_has_ids = 1), + // and the string it then PREPAREs is the same UPDATE / DELETE text. + // + // So on every database this heals, the two revisions executed the identical statements. + // The guard only diverges on the post-2026-06-15-mempool-raw-strings schema, where the + // old form could not have been recorded as applied in the first place. + // + // The check to re-run before extending this entry to a new database: if a row for this + // file can ever be present WITHOUT the migration having completed (a runner that stamps + // before applying, or a hand-inserted ledger row), the argument above does not carry and + // the schema must be reconciled instead. '2026-05-28-unique-index-tables.sql': { - from: '8845b9addc0990b0433f8862969b57cb472535474b4b4d5576c408db777b57ce', // 50a5e83..7817e6c^ + from: [ + 'e1f7df7973881b6fcaa5535fe5aca86b82bb7f45fa4e7e5fdcf9c5859c468207', // 8151979..50a5e83^ + '8845b9addc0990b0433f8862969b57cb472535474b4b4d5576c408db777b57ce', // 50a5e83..7817e6c^ + ], to: '4f7f53ea5423d5ad50e0a2136243dab9e215033e6a110c7b47e66ba5361d44c2', // 7817e6c (HEAD) }, // THE ONE ENTRY THAT DOES NOT MEET THE BYTE-IDENTICAL-SQL CONTRACT ABOVE, said plainly diff --git a/src/protocol/constants.js b/src/protocol/constants.js index e2a5d26..c8323ef 100644 --- a/src/protocol/constants.js +++ b/src/protocol/constants.js @@ -355,11 +355,12 @@ const ORACLE_FEE_OUTPUT_ACTIVATION = { // single-pick therefore stays live BELOW the gate, and a re-decode of pre-flag-day history // reproduces exactly what the fleet wrote live. // -// null means DISARMED (never active), the fail-closed default: mainnet and testnet keep the -// legacy single-pick until that network's maintainers ratify an instant, chosen with the -// fleet's upgrade state in hand, because arming it too early forks the chain and arming it in -// the past rewrites agreed history. regtest holds no agreed history (its chains are recreated -// per run), so it is genesis-on and exercises the set path in the regtest venues. +// mainnet is ARMED at the base gate's own instant by the 2026-09-09 ruling, the earliest the +// ordering above permits: the indexed mainnet history holds 0 dispensers (measured +// 2026-09-09), so set capture persists exactly the output set the legacy single-pick did and +// the arm rewrites no agreed history. A from-genesis OLD-vs-ON replay witness per chain is the +// proof. regtest holds no agreed history (its chains are recreated per run), so it is +// genesis-on and exercises the set path in the regtest venues. // // DEPLOY DEADLINE, once an instant is armed: EVERY decoder on that network MUST be running the // armed value before the instant, or the fleet splits on the first refill of a source holding @@ -369,7 +370,7 @@ const ORACLE_FEE_OUTPUT_ACTIVATION = { // keeps the two copies in lockstep and refuses a value that precedes // ORACLE_FEE_OUTPUT_ACTIVATION. const ORACLE_FEE_SET_CAPTURE_ACTIVATION = { - mainnet: null, // DISARMED: awaiting the operator's ratified per-network instant + mainnet: 1786060800, // ARMED by the 2026-09-09 ruling at its base gate's own instant, the earliest the ordering above permits; identity on the indexed mainnet history (0 dispensers, measured 2026-09-09) // ARMED AT GENESIS (instant 0 = always in force), operator-ratified 2026-08-18 under the // pre-launch ruling that every feature must be ACTIVE on testnet. This gate fixes a defect // that spends a payer native coin and gives nothing back, so a public testnet WILL hit it. @@ -405,11 +406,12 @@ const ORACLE_FEE_SET_CAPTURE_ACTIVATION = { // The legacy block-start soft-expire therefore stays live BELOW the gate, and a re-decode of // pre-flag-day history reproduces exactly what the fleet wrote live. // -// null means DISARMED (never active), the fail-closed default: mainnet and testnet keep the -// legacy block-start expiry until that network's maintainers ratify an instant, chosen with the -// fleet's upgrade state in hand, because arming it too early forks the chain and arming it in -// the past rewrites agreed history. regtest holds no agreed history (its chains are recreated -// per run), so it is genesis-on and exercises the realigned path in the regtest venues. +// mainnet is ARMED at genesis (instant 0) by the 2026-09-09 ruling: the indexed mainnet history +// holds 0 dispensers and 0 dispenses (measured 2026-09-09), so no block ever carried an expiry +// boundary the realigned soft-expire could move and the arm rewrites no agreed history. A +// from-genesis OLD-vs-ON replay witness per chain is the proof. regtest holds no agreed history +// (its chains are recreated per run), so it is genesis-on and exercises the realigned path in +// the regtest venues. // // DEPLOY DEADLINE, once an instant is armed: EVERY decoder on that network MUST be running the // armed value before the instant, or the fleet splits on the first block whose header time @@ -418,7 +420,7 @@ const ORACLE_FEE_SET_CAPTURE_ACTIVATION = { // Vendored byte-equal into xchain-decoder/src/protocol/constants.js; the conformance suite // keeps the two copies in lockstep. const DISPENSER_EXPIRY_REALIGN_ACTIVATION = { - mainnet: null, // DISARMED: awaiting the operator's ratified per-network instant + mainnet: 0, // ARMED at genesis by the 2026-09-09 ruling: identity on the indexed mainnet history (0 dispensers, 0 dispenses, measured 2026-09-09) // ARMED AT GENESIS (instant 0 = always in force), operator-ratified 2026-08-18 under the // pre-launch ruling that every feature must be ACTIVE on testnet. This gate fixes a defect // that spends a payer native coin and gives nothing back, so a public testnet WILL hit it. @@ -458,10 +460,10 @@ const DISPENSER_EXPIRY_REALIGN_ACTIVATION = { // capture set therefore stays live BELOW the gate, and a re-decode of pre-flag-day history // reproduces exactly what the fleet wrote. // -// null means DISARMED (never active), the fail-closed default: mainnet keeps the unwidened -// capture set until that network's maintainers ratify an instant, chosen with the fleet's -// upgrade state in hand, because arming it too early forks the chain and arming it in the -// past rewrites agreed history. +// mainnet is ARMED at genesis (instant 0) by the 2026-09-09 ruling: the indexed mainnet +// history holds 0 dispensers and 0 dispenses (measured 2026-09-09), so the widened capture +// set admits no output the unwidened one missed and the arm rewrites no agreed history. A +// from-genesis OLD-vs-ON replay witness per chain is the proof. // // DEPLOY DEADLINE, once an instant is armed: EVERY decoder on that network MUST be running // the armed value before the instant, or the fleet splits on the first block whose header @@ -470,7 +472,7 @@ const DISPENSER_EXPIRY_REALIGN_ACTIVATION = { // Vendored byte-equal into xchain-decoder/src/protocol/constants.js; the conformance suite // keeps the two copies in lockstep. const DISPENSER_CANCEL_GRACE_ACTIVATION = { - mainnet: null, // DISARMED: awaiting the operator's ratified per-network instant + mainnet: 0, // ARMED at genesis by the 2026-09-09 ruling: identity on the indexed mainnet history (0 dispensers, 0 dispenses, measured 2026-09-09) // ARMED AT GENESIS (instant 0 = always in force), matching the sibling // DISPENSER_EXPIRY_REALIGN_ACTIVATION under the pre-launch ruling that every feature must // be ACTIVE on testnet. This gate closes a defect that spends a payer's native coin and diff --git a/src/protocol/indexerBatchLimits.js b/src/protocol/indexerBatchLimits.js index 59730c4..496f955 100644 --- a/src/protocol/indexerBatchLimits.js +++ b/src/protocol/indexerBatchLimits.js @@ -77,7 +77,7 @@ const COMMAND_WEIGHTS = { // on wherever the decoder's capture gate is: mainnet capture is armed while this instant is // still the house sentinel. null means DISARMED, which is inactive at every block time. const COST_WEIGHTING_ACTIVATION = { - "mainnet": 9999999999, + "mainnet": 0, "testnet": 0, "regtest": 0, }; diff --git a/src/shutdown.js b/src/shutdown.js new file mode 100644 index 0000000..0c85bde --- /dev/null +++ b/src/shutdown.js @@ -0,0 +1,187 @@ +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * + * XChain Decoder - Graceful shutdown + * + * Bounded, idempotent drain for SIGTERM/SIGINT, the same shape as the + * indexer's src/shutdown.js. The Dockerfile CMD runs node as PID 1, so + * `docker stop` delivers SIGTERM here. + * + * Before this file the handler in api.js only set the decoder's stopFlag. The + * parse loop did break at its next block boundary, but the Express listener + * and the MariaDB pool kept the event loop alive and nothing called exit, so + * the process parked with the loop stopped until docker's SIGKILL: every stop + * of a decoder, ever, ended in exit 137 (measured by an operator with + * `docker stop -t 180` on a BTC mainnet stack, 2026-09-10). A killed decoder + * mid-rollback is the case that matters, because an interrupted rollback is + * what resets the dispenser purge budget. + * + * Registering a handler REMOVES node's default terminate, so the handler + * carries its own hard-exit timer: a drain that hangs must still end the + * process, or a stop becomes a container that lingers under any supervisor + * with a long grace period, which is strictly worse than the kill. + * + ********************************************************************/ + +// Hard-exit budget for the whole drain. xchain-node stops a decoder with a +// 120 s budget (and stamps it on the container as --stop-timeout), so the +// default sits under that: an overrun that ends in our own logged exit is +// diagnosable, one that ends in the daemon's SIGKILL is not. On a container +// created before the budget existed docker's ten seconds still applies and +// this timer never gets to fire; nothing here can change that from inside. +// SHUTDOWN_TIMEOUT_MS overrides for a slow chain. +const DEFAULT_SHUTDOWN_TIMEOUT_MS = 100000; + +function resolveTimeoutMs(timeoutMs, env){ + if(Number.isFinite(timeoutMs) && timeoutMs > 0) return timeoutMs; + const raw = parseInt((env || process.env).SHUTDOWN_TIMEOUT_MS, 10); + return (Number.isFinite(raw) && raw > 0) ? raw : DEFAULT_SHUTDOWN_TIMEOUT_MS; +} + +// Close an http.Server and resolve once it has stopped listening. Idle keep-alive +// sockets would otherwise hold close() open indefinitely while no request is in +// flight, so they are dropped explicitly; requests already being served finish. +function closeServer(server){ + return new Promise((resolve) => { + if(!server || typeof server.close !== 'function') return resolve(); + let settled = false; + const done = () => { if(!settled){ settled = true; resolve(); } }; + try { + server.close(done); + if(typeof server.closeIdleConnections === 'function') server.closeIdleConnections(); + } catch(_){ + done(); + } + }); +} + +// Best-effort close of a set of Database handles, deduped by identity. A pool +// that refuses to close must not abort the rest of the drain. +async function closeDatabases(handles, log){ + const logger = log || console; + const seen = new Set(); + for(const db of (handles || [])){ + if(!db || typeof db.close !== 'function' || seen.has(db)) continue; + seen.add(db); + try { await db.close(); } + catch(err){ logger.warn('Shutdown: closing a database pool failed: ' + (err && err.message ? err.message : err)); } + } +} + +/** + * Build an idempotent signal handler that runs `drain` under a hard-exit bound. + * + * @param {object} opts + * @param {function} opts.drain async work to finish before exiting + * @param {number} [opts.timeoutMs] hard-exit budget (default SHUTDOWN_TIMEOUT_MS / 100000) + * @param {function} [opts.exit] process-exit seam (tests pass their own) + * @param {object} [opts.log] console-shaped logger + * @returns {function(string): void} handler to register on SIGTERM / SIGINT + */ +function createShutdown({ drain, timeoutMs, exit, log } = {}){ + const onExit = exit || ((code) => process.exit(code)); + const logger = log || console; + const budget = resolveTimeoutMs(timeoutMs); + let signalled = false; + + return function shutdown(signal){ + // A second signal must not restart the sequence: re-entering would call + // stop() and close pools underneath a drain already using them. + if(signalled){ + logger.log('Shutdown already in progress; ignoring ' + (signal || 'signal') + '.'); + return; + } + signalled = true; + logger.log('Received ' + (signal || 'signal') + ', draining (hard exit in ' + budget + 'ms)...'); + + let finished = false; + const timer = setTimeout(() => { + if(finished) return; + finished = true; + // Non-zero: the drain did NOT complete, so work was cut off exactly as a + // SIGKILL would have cut it. A clean drain below exits 0. + logger.error('Shutdown drain exceeded ' + budget + 'ms; exiting hard.'); + onExit(1); + }, budget); + + Promise.resolve().then(() => drain()).then( + () => { + if(finished) return; + finished = true; + clearTimeout(timer); + logger.log('Shutdown drain complete; exiting.'); + onExit(0); + }, + (err) => { + if(finished) return; + finished = true; + clearTimeout(timer); + logger.error('Shutdown drain failed:', err); + onExit(1); + } + ); + }; +} + +/** + * The decoder's drain, as its own function so the exit path is unit-testable. + * + * Order is load-bearing: + * 1. flip the health flag FIRST: stop() only sets stopFlag and the parse loop + * can take a whole block to notice, and /live must not answer 200 for a + * decoder that is leaving. + * 2. stop() the decoder (stopFlag; the loop clears its own mempool interval + * on the way out). + * 3. drain the HTTP server and the parse loop together; the loop breaks at + * the top of its iteration, never mid-transaction. + * 4. close the two DB pools LAST, since both of the above still need them. + * + * The wait on step 3 is unbounded HERE and bounded by the caller's hard-exit + * timer, because both ways it can overrun (a block or a rollback slower than + * the budget, or a boot still inside the DB connect retry that never entered + * the loop) should end in a logged non-zero exit, not a clean one. + * + * @param {object} opts + * @param {object} opts.decoder XChainDecoder instance + * @param {object} opts.server http.Server returned by app.listen() + * @param {Promise} [opts.loopSettled] promise that settles when start()'s loop exits + * @param {function} [opts.onDraining] flips the api-local decoderRunning flag + * @param {object} [opts.log] console-shaped logger + */ +function createDecoderDrain({ decoder, server, loopSettled, onDraining, log } = {}){ + const logger = log || console; + return async function drain(){ + if(typeof onDraining === 'function') onDraining(); + if(decoder && typeof decoder.stop === 'function') decoder.stop(); + + await Promise.all([ + closeServer(server), + // start() resolves when the parse loop breaks on stopFlag. It is already + // .catch()'d at the call site (a fatal decoder error exits 1 there), so a + // rejection here is that same handled error and must not fail the drain. + Promise.resolve(loopSettled).catch(() => {}) + ]); + + await closeDatabases(decoder ? [decoder.db, decoder.mempoolDb] : [], logger); + }; +} + +module.exports = { + DEFAULT_SHUTDOWN_TIMEOUT_MS, + resolveTimeoutMs, + closeServer, + closeDatabases, + createShutdown, + createDecoderDrain +}; diff --git a/test/chaos/CE08-signalHandling.chaos.js b/test/chaos/CE08-signalHandling.chaos.js index 2851e60..0bedf27 100644 --- a/test/chaos/CE08-signalHandling.chaos.js +++ b/test/chaos/CE08-signalHandling.chaos.js @@ -144,10 +144,12 @@ describe('CE-08: Signal Handling and Graceful Shutdown', function () { assert.ok(/decoder\.start\(\)\s*\.then\(/.test(apiSource), 'decoder.start() should have a .then() that observes a clean loop exit') - const shutdownBody = apiSource.slice(apiSource.indexOf('const shutdown = () =>'), + const shutdownBody = apiSource.slice(apiSource.indexOf('createDecoderDrain('), apiSource.indexOf("process.on('SIGTERM'")) assert.ok(shutdownBody.includes('decoderRunning = false'), - 'shutdown() should mark the decoder not-running before stopping it') + 'the drain should mark the decoder not-running before stopping it') + // The drain itself (flag first, stop, listener and loop, pools last, hard + // exit) is pinned behaviourally in test/unit/shutdown.test.js. // Whether /live actually turns 503 on a silent heartbeat is pinned // behaviourally against the shipped route in // test/unit/decoderLiveHeartbeat.test.js. A grep for `isPollSilent` here would diff --git a/test/e2e/dispenserLifecycle.e2e.js b/test/e2e/dispenserLifecycle.e2e.js index 432a5b0..9b92e51 100644 --- a/test/e2e/dispenserLifecycle.e2e.js +++ b/test/e2e/dispenserLifecycle.e2e.js @@ -113,8 +113,9 @@ describe('E2E: DISPENSER Lifecycle', function () { // sweep runs AFTER the block's own transactions, exactly where the indexer's // processExpirations sits. An already-past expiration is therefore stamped by // the block that CARRIES the create, not the one after it. (Below the gate the - // sweep ran first and a create could never be expired by its own block, which - // is the legacy behavior mainnet/testnet keep until an instant is ratified.) + // sweep ran first and a create could never be expired by its own block. Every + // network is armed at genesis since the 2026-09-09 ruling, so that legacy path + // is what a re-decode runs only on a network armed mid-chain.) // // Expiry is a SOFT expire, not a delete: db.deleteOpenDispensers stamps // the expiring block height into expired_block_index so a reorg can diff --git a/test/unit/batchLimitsVendoring.test.js b/test/unit/batchLimitsVendoring.test.js index cee8889..ef0f185 100644 --- a/test/unit/batchLimitsVendoring.test.js +++ b/test/unit/batchLimitsVendoring.test.js @@ -47,6 +47,7 @@ const { BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION, subCommandCostWeight, subCommandLimitKey, subCommandTick, + isBatchCostWeightingActive, CHILD_ISSUE_KEY } = require('../../src/batchSubCommandCapture.js'); const ACTION_ALIASES = require('../../src/actionAliases.js'); const sync = require('../tools/sync-batch-limits.js'); @@ -56,6 +57,12 @@ const CORPUS = require('../fixtures/regtestBatchCorpus.json'); const REQUIRE_SIBLINGS = process.env.XCHAIN_REQUIRE_SIBLINGS === '1'; const T0 = 1700000000; +// One over-budget wire the gate blocks can drive without reaching into tier 3's vectors: +// 10 sub-commands, well under the 250-COUNT cap, weighing 271 against the 250 budget, so +// only the WEIGHT budget can ever suppress it. Same shape as tier 3's '9x EXECUTE + SEND'. +const WEIGHT_PROBE = 'BATCH|0|SEND|0|BTC|TICK|1|addr;' + + Array.from({ length: 9 }, () => 'EXECUTE|0|1|a').join(';'); + function siblingOrSkip(ctx, file) { if (fs.existsSync(file)) return true; if (REQUIRE_SIBLINGS) @@ -335,17 +342,67 @@ describe('BATCH limit vendoring and cross-repo conformance', function () { 'the indexer still dispatches'); }); - it('does NOT assume the weighting flag is on wherever capture is, which is why it is gated', function () { - // The counter-example that killed the ordering shortcut, pinned so it stays a - // counter-example: mainnet capture is armed and mainnet weighting is not. An - // ungated budget would suppress mainnet capture today. + it('never applies the budget where the indexer would not: capture is the narrower gate', function () { + // The ordering that protects the money-bearing direction, re-derived after the + // 2026-09-09 genesis arm moved mainnet weighting from the house sentinel to 0. + // + // The old shape of this test pinned "mainnet capture is armed and mainnet + // weighting is not", which was true and is not any more. What replaces it is + // stronger, because it holds in the direction that costs money rather than + // merely being a fact about two numbers: + // + // * the indexer's budget is a strict refinement of BATCH_ISSUANCE_LIMITS. + // src/actions/batch.js reads its BATCH_COST_WEIGHTING verdict ONLY inside + // `if(limitsActive)` blocks, so below that gate's mainnet instant no bound + // runs at all, whatever the weighting instant says; + // * this decoder cannot suppress there either, because captureCommands exits + // with the un-expanded passthrough while the CAPTURE gate is inactive, and + // mainnet capture arms at that same instant. + // + // So the window where the vendored weighting instant reads "on" but the indexer + // applies no budget is exactly the window where this module captures nothing to + // suppress. Under-capture, the direction that loses a settlement output, is + // impossible in it. Both halves are driven, not asserted about the constants. const captureGate = BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION.mainnet; const weightGate = VENDORED_MODULE.COST_WEIGHTING_ACTIVATION.mainnet; if (captureGate === null || typeof weightGate !== 'number') return; - assert.ok(captureGate < weightGate, - 'mainnet capture (' + captureGate + ') is no longer earlier than mainnet ' + - 'weighting (' + weightGate + '); re-read isBatchCostWeightingActive before ' + - 'relying on the gate, and re-derive whether the budget may now be unconditional'); + + assert.ok(weightGate <= captureGate, + 'mainnet weighting (' + weightGate + ') now arms AFTER capture (' + captureGate + + '); a batch could then be captured with the budget still off here while the ' + + 'indexer applied it, which is over-capture in the other direction'); + + // Inside the window: the vendored weighting gate reads active, and capture does + // not, so no batch reaches the budget. + const inside = captureGate - 1; + assert.strictEqual(isBatchCostWeightingActive('mainnet', inside), true, + 'the vendored mainnet weighting instant is 0, so it must read active below capture'); + assert.deepStrictEqual( + captureCommands(WEIGHT_PROBE, 'mainnet', inside), [WEIGHT_PROBE], + 'capture must still be OFF inside the window: an over-budget batch that ' + + 'reached the budget here would be suppressed while the indexer dispatched it'); + + // At and above the instant both gates are on together, which is the state the + // tier 3 block drives against the real handler. + assert.strictEqual(isBatchCostWeightingActive('mainnet', captureGate), true); + assert.deepStrictEqual(captureCommands(WEIGHT_PROBE, 'mainnet', captureGate), [], + 'at the shared instant the mirror must suppress the over-budget batch, ' + + 'because the indexer rejects it there'); + }); + + it('testnet and regtest have no such window: capture and weighting both arm at genesis', function () { + // The two networks the window argument does not need, pinned so a future + // per-network re-pin cannot open one quietly. + for (const network of ['testnet', 'regtest']) { + const captureGate = BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION[network]; + const weightGate = VENDORED_MODULE.COST_WEIGHTING_ACTIVATION[network]; + assert.strictEqual(captureGate, 0, network + ' capture is no longer genesis-active'); + assert.strictEqual(weightGate, 0, network + ' weighting is no longer genesis-active'); + assert.strictEqual(isBatchCostWeightingActive(network, 0), true, + network + ' must weigh from block 0'); + assert.deepStrictEqual(captureCommands(WEIGHT_PROBE, network, 0), [], + network + ' must suppress the over-budget batch from block 0'); + } }); }); @@ -371,7 +428,14 @@ describe('BATCH limit vendoring and cross-repo conformance', function () { wire: 'BATCH|0|' + Array.from({ length: 10 }, () => 'AIRDROP|0|BTC|TICK|1|a').join(';') }, ]; - const MAINNET_LIVE = 1800000000; // above mainnet capture, below the weighting sentinel + // Above mainnet capture, which since the 2026-09-09 genesis arm is also above the + // point where the indexer's own weight budget becomes reachable (its BATCH_COST_ + // WEIGHTING verdict is read only inside the BATCH_ISSUANCE_LIMITS guard, and that + // gate's mainnet instant is the capture instant). Both sides weigh here. + const MAINNET_LIVE = 1800000000; + // Inside the inverted window instead: the weighting instant is 0 so the vendored + // gate reads active, but capture is off here and the indexer applies no bound. + const MAINNET_WINDOW = BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION.mainnet - 1; it('suppresses an over-budget batch on regtest, where the handler rejects it whole', async function () { if (!siblingOrSkip(this, sync.INDEXER_BATCH)) return; @@ -388,19 +452,44 @@ describe('BATCH limit vendoring and cross-repo conformance', function () { } }); - it('still captures an over-budget batch on MAINNET, where the flag is unarmed', async function () { - // The under-capture control, and the reason the rule is gated instead of - // unconditional. Pre-gate reasoning would have suppressed these. + it('still captures an over-budget batch inside the inverted MAINNET window', async function () { + // The under-capture control, re-aimed at the window the 2026-09-09 genesis arm + // opened. Mainnet BATCH_COST_WEIGHTING is now 0, so the vendored gate reads + // active below the capture instant; the real handler applies NO bound there, + // because it reads that verdict only inside its BATCH_ISSUANCE_LIMITS guard and + // that gate arms at the capture instant. This is the case that would lose a + // settlement output if the mirror ever suppressed on the weighting instant alone. if (!siblingOrSkip(this, sync.INDEXER_BATCH)) return; for (const vector of WEIGHT_VECTORS) { const status = await indexerStatus(vector.wire, - { network: 'mainnet', blockTime: MAINNET_LIVE }); + { network: 'mainnet', blockTime: MAINNET_WINDOW }); assert.strictEqual(status, 'valid', - vector.name + ': premise wrong, mainnet handler said ' + status); - const view = captureCommands(vector.wire, 'mainnet', MAINNET_LIVE); - assert.strictEqual(view.length, subCommandsOf(vector.wire).length, + vector.name + ': premise wrong, mainnet handler said ' + status + + ' inside the window; the budget is no longer nested under BATCH_ISSUANCE_LIMITS'); + // Capture is off here, so the mirror hands back the un-expanded batch rather + // than suppressing it. Nothing the handler dispatches is dropped. + assert.deepStrictEqual(captureCommands(vector.wire, 'mainnet', MAINNET_WINDOW), + [vector.wire], 'UNDER-CAPTURE on mainnet: the mirror suppressed ' + vector.name + - ', which the real handler dispatches in full'); + ' inside the window, which the real handler dispatches in full'); + } + }); + + it('and agrees with the handler ABOVE the shared instant, where both weigh', async function () { + // The other side of the same boundary, and the state mainnet is actually in + // today. Once capture is on, BATCH_ISSUANCE_LIMITS is on too, so the indexer's + // budget is reachable and both sides must reach the same verdict. Without this + // the case above would also pass if the mirror had simply stopped suppressing. + if (!siblingOrSkip(this, sync.INDEXER_BATCH)) return; + for (const vector of WEIGHT_VECTORS) { + const status = await indexerStatus(vector.wire, + { network: 'mainnet', blockTime: MAINNET_LIVE }); + assert.strictEqual(status, 'invalid: COMMAND (limit)', + vector.name + ': the mainnet handler said ' + status + ' above the ' + + 'capture instant, where the weight budget is reachable'); + assert.deepStrictEqual(captureCommands(vector.wire, 'mainnet', MAINNET_LIVE), [], + 'OVER-CAPTURE on mainnet: the mirror captured ' + vector.name + + ', which the real handler rejects whole'); } }); diff --git a/test/unit/dispenserCancelGrace.test.js b/test/unit/dispenserCancelGrace.test.js index 8aca4e1..8f47c9a 100644 --- a/test/unit/dispenserCancelGrace.test.js +++ b/test/unit/dispenserCancelGrace.test.js @@ -35,7 +35,8 @@ const sinon = require('sinon') const XChainDecoder = require('../../src/XChainDecoder') const Database = require('../../src/db.js') -const { DISPENSER_CANCEL_GRACE_SECONDS, +const { DISPENSER_CANCEL_GRACE_ACTIVATION, + DISPENSER_CANCEL_GRACE_SECONDS, cancelGraceFloor } = require('../../src/dispenserCancelGrace') const PREV_WIRE = Buffer.from( @@ -219,11 +220,22 @@ describe('dispenser cancellation grace: decoder capture outlasts the indexer fil }) it('keeps the unwidened capture set below the flag-day (the other side of the gate)', async () => { - // Same blocks, same model, DISARMED network. This is the behavior the fleet runs today - // and the behavior a from-genesis re-decode of pre-flag-day history must reproduce. + // Same blocks, same model, gate DISARMED. Every network in the map is armed at genesis + // since the 2026-09-09 ruling, so the below-gate branch is reached by disarming mainnet + // in place for the length of this test. The branch stays live code: it is what a + // from-genesis re-decode runs on any network that arms mid-chain, and dropping the + // assertion would let the widened set become unconditional without a test noticing. const payAt = EXPIRATION + 1800 const model = fundedCancelledDispenser() - await runTwoBlocks('mainnet', EXPIRATION + 1, payAt, model) + const saved = DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet + DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet = null + try { + await runTwoBlocks('mainnet', EXPIRATION + 1, payAt, model) + } finally { + DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet = saved + } + assert.strictEqual(DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet, 0, + 'the map must be back to the genesis arm after the probe') assert.strictEqual(model.rows[0].expiredBlockIndex, 0) const payLoad = model.captureLoads[1] @@ -233,6 +245,21 @@ describe('dispenser cancellation grace: decoder capture outlasts the indexer fil 'below the gate the expired dispenser stays out of the capture set') }) + it('carries the grace on mainnet at genesis, the state the 2026-09-09 ruling armed', async () => { + // The armed mainnet path driven through the real block loop, not just the helper: the + // same cancelled dispenser is captured for a payment inside the indexer's fill window. + const payAt = EXPIRATION + 1800 + const model = fundedCancelledDispenser() + await runTwoBlocks('mainnet', EXPIRATION + 1, payAt, model) + + assert.strictEqual(model.rows[0].expiredBlockIndex, 0, + 'block 0 must have soft-expired the dispenser, or this test proves nothing') + const payLoad = model.captureLoads[1] + assert.strictEqual(payLoad.floor, payAt - DISPENSER_CANCEL_GRACE_SECONDS) + assert.ok(payLoad.set.has(ADDR), + 'mainnet is armed at genesis, so the grace carries the address into the capture set') + }) + it('closes capture once the indexer can no longer settle a fill', async () => { // The grace is a window, not an amnesty: past expiration + grace the address leaves the // capture set, and by then the indexer stopped matching the dispenser long ago. diff --git a/test/unit/dispenserCancelGraceActivation.test.js b/test/unit/dispenserCancelGraceActivation.test.js index 600a503..683dc24 100644 --- a/test/unit/dispenserCancelGraceActivation.test.js +++ b/test/unit/dispenserCancelGraceActivation.test.js @@ -20,11 +20,14 @@ // matching what the fleet wrote live; // * arming it on a network whose decoders are not all running the value forks the fleet at // the first block that passes a cancelled dispenser's expiration. -// So mainnet is DISARMED (null) until the operator ratifies an instant, and the helper fails -// closed on anything that is not a number. +// mainnet is ARMED AT GENESIS (instant 0) by the 2026-09-09 ruling. Arming it there rewrites +// nothing: the indexed mainnet history holds 0 dispensers and 0 dispenses (measured +// 2026-09-09), so the widened capture set admits no output the unwidened one missed. The helper +// still fails closed on anything that is not a number, which is what the null sentinel remains +// for on any network that has not armed. // // Two tiers, so a one-sided edit fails somewhere no matter which checkout is present: -// 1. PIN - the vendored map has the disarmed/genesis-on shape, in this repo alone. +// 1. PIN - the vendored map has the genesis-on shape on every network, in this repo alone. // 2. DOCS - it is value-identical to the canonical map in // xchain-documentation/protocol/constants.js. // Tier 2 skips when the sibling checkout is absent (standalone deploy); set @@ -67,14 +70,26 @@ function indexerCloseDelay(){ describe('DISPENSER_CANCEL_GRACE_ACTIVATION conformance', function () { - it('keeps mainnet DISARMED, with testnet and regtest genesis-on', function () { - // Teeth for the ratification requirement: a number on MAINNET means someone armed a - // consensus boundary without the operator's ratified instant. - assert.strictEqual(DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet, null); + it('arms mainnet at genesis by the 2026-09-09 ruling, with testnet and regtest genesis-on', function () { + // Teeth for the ruling: mainnet sits at instant 0, which is identity on the indexed + // mainnet history (0 dispensers, 0 dispenses, measured 2026-09-09). Any other mainnet + // value re-introduces a boundary block the fleet could split on, so it fails here. + assert.strictEqual(DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet, 0); assert.strictEqual(DISPENSER_CANCEL_GRACE_ACTIVATION.testnet, 0); assert.strictEqual(DISPENSER_CANCEL_GRACE_ACTIVATION.regtest, 0); }); + it('mainnet carries the grace from block time 0 upward, floor and all', function () { + // The behaviour the arm buys: every mainnet block, the genesis instant included, keeps + // a just-expired dispenser in the capture set for one grace window, so a cancelled + // dispenser can never take a payment the decoder drops. + assert.strictEqual(isDispenserCancelGraceActive('mainnet', 0), true); + assert.strictEqual(isDispenserCancelGraceActive('mainnet', 1786060800), true); + assert.strictEqual(isDispenserCancelGraceActive('mainnet', 4000000000), true); + assert.strictEqual(cancelGraceFloor('mainnet', 4000000000), + 4000000000 - DISPENSER_CANCEL_GRACE_SECONDS); + }); + it('is value-identical to the canonical map in xchain-documentation', function () { if (!siblingOrSkip(this, DOCS_CONSTANTS)) return; const canon = require(DOCS_CONSTANTS).DISPENSER_CANCEL_GRACE_ACTIVATION; @@ -89,13 +104,23 @@ describe('DISPENSER_CANCEL_GRACE_ACTIVATION conformance', function () { 'the decoder fleet at the first block that passes a cancelled dispenser expiration'); }); - it('a DISARMED network is inactive at every block time, including absurd ones', function () { - // A `time >= null` coercion would read 0 and arm mainnet from genesis, which is the - // failure this pins. - assert.strictEqual(isDispenserCancelGraceActive('mainnet', 0), false); - assert.strictEqual(isDispenserCancelGraceActive('mainnet', 1786060800), false); - assert.strictEqual(isDispenserCancelGraceActive('mainnet', 4000000000), false); - assert.strictEqual(cancelGraceFloor('mainnet', 4000000000), null); + it('a DISARMED (null) network is inactive at every block time, including absurd ones', function () { + // No network carries the null sentinel now that mainnet is armed, so disarm one in + // place for the length of this test and drive the REAL helper. A `time >= null` + // coercion would read 0 and widen the capture set from genesis on a network whose + // fleet never armed it, which is the failure this pins. + const saved = DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet; + DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet = null; + try { + assert.strictEqual(isDispenserCancelGraceActive('mainnet', 0), false); + assert.strictEqual(isDispenserCancelGraceActive('mainnet', 1786060800), false); + assert.strictEqual(isDispenserCancelGraceActive('mainnet', 4000000000), false); + assert.strictEqual(cancelGraceFloor('mainnet', 4000000000), null); + } finally { + DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet = saved; + } + assert.strictEqual(DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet, 0, + 'the map must be back to the genesis arm after the probe'); }); it('testnet and regtest are active from genesis', function () { @@ -119,11 +144,11 @@ describe('DISPENSER_CANCEL_GRACE_ACTIVATION conformance', function () { assert.strictEqual(cancelGraceFloor('regtest', NaN), null); }); - it('flips exactly at the armed instant once a network IS armed (>= semantics)', function () { - // The map is disarmed today, so arm a network in place for the length of this test and + it('flips exactly at the armed instant when a network is armed mid-chain (>= semantics)', function () { + // Mainnet arms at 0, so move it to a mid-chain instant for the length of this test and // drive the REAL helper (the module reads the map per call, so the mutation is - // visible). This pins the boundary the operator will ratify onto: >=, so the block AT - // the instant already carries the grace, matching every protocol_changes gate. + // visible). This pins the boundary semantics any later arm inherits: >=, so the block + // AT the instant already carries the grace, matching every protocol_changes gate. const ARMED = 1789430400; const saved = DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet; DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet = ARMED; @@ -139,8 +164,10 @@ describe('DISPENSER_CANCEL_GRACE_ACTIVATION conformance', function () { } finally { DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet = saved; } - assert.strictEqual(isDispenserCancelGraceActive('mainnet', ARMED), false, - 'the map must be back to DISARMED after the probe'); + assert.strictEqual(DISPENSER_CANCEL_GRACE_ACTIVATION.mainnet, 0, + 'the map must be back to the genesis arm after the probe'); + assert.strictEqual(isDispenserCancelGraceActive('mainnet', ARMED - 1), true, + 'and the restored genesis arm covers the block the probe held below its instant'); }); it('the floor is exactly one grace window below the block time', function () { diff --git a/test/unit/dispenserExpiryRealignActivation.test.js b/test/unit/dispenserExpiryRealignActivation.test.js index e45a39b..39eb898 100644 --- a/test/unit/dispenserExpiryRealignActivation.test.js +++ b/test/unit/dispenserExpiryRealignActivation.test.js @@ -19,11 +19,14 @@ // what the fleet wrote live; // * arming it on a network whose decoders are not all running the value forks the fleet at the // first boundary block. -// So the map is DISARMED (null) on mainnet and testnet until the operator ratifies a per-network -// instant, and the helper fails closed on anything that is not a number. +// mainnet is ARMED AT GENESIS (instant 0) by the 2026-09-09 ruling. Arming it there rewrites +// nothing: the indexed mainnet history holds 0 dispensers and 0 dispenses (measured 2026-09-09), +// so no mainnet block ever carried an expiry boundary the realigned soft-expire could move. The +// helper still fails closed on anything that is not a number, which is what the null sentinel +// remains for on any network that has not armed. // // Two tiers, so a one-sided edit fails somewhere no matter which checkout is present: -// 1. PIN - the vendored map has the disarmed/genesis-on shape, in this repo alone. +// 1. PIN - the vendored map has the genesis-on shape on every network, in this repo alone. // 2. DOCS - it is value-identical to the canonical map in // xchain-documentation/protocol/constants.js. // Tier 2 skips when the sibling checkout is absent (standalone deploy); set @@ -51,16 +54,26 @@ function siblingOrSkip(ctx, file){ describe('DISPENSER_EXPIRY_REALIGN_ACTIVATION conformance', function () { - it('keeps mainnet DISARMED, with testnet and regtest genesis-on', function () { - // Teeth for the ratification requirement: a number on MAINNET means someone armed a - // consensus boundary without the operator's ratified instant. Testnet was ratified at + it('arms mainnet at genesis by the 2026-09-09 ruling, with testnet and regtest genesis-on', function () { + // Teeth for the ruling: mainnet sits at instant 0, which is identity on the indexed + // mainnet history (0 dispensers, 0 dispenses, measured 2026-09-09). Any other mainnet + // value re-introduces a boundary block, so it has to fail here. Testnet was ratified at // instant 0 on 2026-08-18 (pre-launch, every feature active on testnet), which is safe // only because testnet decoder/indexer state is rebuilt from the chain before launch. - assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.mainnet, null); + assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.mainnet, 0); assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.testnet, 0); assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.regtest, 0); }); + it('mainnet is realigned from block time 0 upward, with no boundary block left', function () { + // The behaviour the arm buys: every mainnet block, including the genesis instant + // itself, measures expiry where the indexer does, so no header time can fall on a + // side of the gate the fleet disagrees about. + assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 0), true); + assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 1786060800), true); + assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 4000000000), true); + }); + it('is value-identical to the canonical map in xchain-documentation', function () { if (!siblingOrSkip(this, DOCS_CONSTANTS)) return; const canon = require(DOCS_CONSTANTS).DISPENSER_EXPIRY_REALIGN_ACTIVATION; @@ -75,12 +88,22 @@ describe('DISPENSER_EXPIRY_REALIGN_ACTIVATION conformance', function () { 'the decoder fleet at the first boundary block'); }); - it('a DISARMED network is inactive at every block time, including absurd ones', function () { - // Mainnet is the network still carrying the null sentinel. A `time >= null` coercion - // would read 0 and arm it from genesis, which is the failure this pins. - assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 0), false); - assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 1786060800), false); - assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 4000000000), false); + it('a DISARMED (null) network is inactive at every block time, including absurd ones', function () { + // No network carries the null sentinel now that mainnet is armed, so disarm one in + // place for the length of this test and drive the REAL helper. A `time >= null` + // coercion would read 0 and arm the network from genesis, which is the failure this + // pins for whichever network is next added to the map unarmed. + const saved = DISPENSER_EXPIRY_REALIGN_ACTIVATION.mainnet; + DISPENSER_EXPIRY_REALIGN_ACTIVATION.mainnet = null; + try { + assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 0), false); + assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 1786060800), false); + assert.strictEqual(isDispenserExpiryRealignActive('mainnet', 4000000000), false); + } finally { + DISPENSER_EXPIRY_REALIGN_ACTIVATION.mainnet = saved; + } + assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.mainnet, 0, + 'the map must be back to the genesis arm after the probe'); }); it('testnet is active from genesis, so the launch runs the realigned path', function () { @@ -107,11 +130,11 @@ describe('DISPENSER_EXPIRY_REALIGN_ACTIVATION conformance', function () { assert.strictEqual(isDispenserExpiryRealignActive('regtest', 'not-a-time'), false); }); - it('flips exactly at the armed instant once a network IS armed (>= semantics)', function () { - // The map is disarmed today, so arm a network in place for the length of this test and + it('flips exactly at the armed instant when a network is armed mid-chain (>= semantics)', function () { + // Mainnet arms at 0, so move it to a mid-chain instant for the length of this test and // drive the REAL helper (the module reads the map per call, so the mutation is visible). - // This pins the boundary the operator will ratify onto: >=, so the block AT the instant - // is already realigned, matching every protocol_changes gate. + // This pins the boundary semantics any later arm inherits: >=, so the block AT the + // instant is already realigned, matching every protocol_changes gate. const ARMED = 1789430400; const saved = DISPENSER_EXPIRY_REALIGN_ACTIVATION.mainnet; DISPENSER_EXPIRY_REALIGN_ACTIVATION.mainnet = ARMED; @@ -125,7 +148,9 @@ describe('DISPENSER_EXPIRY_REALIGN_ACTIVATION conformance', function () { } finally { DISPENSER_EXPIRY_REALIGN_ACTIVATION.mainnet = saved; } - assert.strictEqual(isDispenserExpiryRealignActive('mainnet', ARMED), false, - 'the map must be back to DISARMED after the probe'); + assert.strictEqual(DISPENSER_EXPIRY_REALIGN_ACTIVATION.mainnet, 0, + 'the map must be back to the genesis arm after the probe'); + assert.strictEqual(isDispenserExpiryRealignActive('mainnet', ARMED - 1), true, + 'and the restored genesis arm covers the block the probe held below its instant'); }); }); diff --git a/test/unit/dispenserOracleFeeOutput.test.js b/test/unit/dispenserOracleFeeOutput.test.js index 86d4049..c391b07 100644 --- a/test/unit/dispenserOracleFeeOutput.test.js +++ b/test/unit/dispenserOracleFeeOutput.test.js @@ -237,12 +237,42 @@ describe('DISPENSER PRICE v1 oracle-fee output capture', function () { outputs: [{ destinationAddress: payTo, vout: 0, amount: '0.00000600' }] }, ]) - // regtest is genesis-on for both gates; mainnet at the base flag-day has capture on - // and set capture still DISARMED, which is the pre-fix behavior to preserve. + // regtest is genesis-on for both gates. mainnet arms set capture at the base gate's own + // instant since the 2026-09-09 ruling, so no mainnet block time sits between the two + // gates any more: the pre-fix single-pick behavior is reached by disarming the set gate + // in place instead. It stays live code for any network that arms mid-chain, and a + // re-decode of pre-flag-day history must still reproduce it. const ABOVE = { network: 'bitcoin-regtest', blockTime: T0 } const BELOW = { network: 'bitcoin-mainnet', blockTime: ORACLE_FEE_OUTPUT_ACTIVATION.mainnet, feeDestination: null } + // Run `fn` with mainnet set capture disarmed, restoring the ruling's armed value even + // if the body throws, so a failure here cannot leak a null into a later test. + async function withSetCaptureDisarmed(fn){ + const saved = ORACLE_FEE_SET_CAPTURE_ACTIVATION.mainnet + ORACLE_FEE_SET_CAPTURE_ACTIVATION.mainnet = null + try { await fn() } + finally { ORACLE_FEE_SET_CAPTURE_ACTIVATION.mainnet = saved } + assert.strictEqual(ORACLE_FEE_SET_CAPTURE_ACTIVATION.mainnet, + ORACLE_FEE_OUTPUT_ACTIVATION.mainnet, + 'the map must be back to the armed instant after the probe') + } + + it('captures the oracle of a NON-top-ranked open dispenser on ARMED mainnet', async () => { + // The state the 2026-09-09 ruling put mainnet in, driven at the armed instant: the + // refill of the older row captures its own oracle, not the top-ranked one's. + const model = new DispenserModel() + const decoder = buildDecoder(twoOpenThenRefill(ORACLE_A), model, + { network: 'bitcoin-mainnet', blockTime: ORACLE_FEE_SET_CAPTURE_ACTIVATION.mainnet, + feeDestination: null }) + + await decoder.start() + + assert.strictEqual(model.rows.length, 2, 'both creates registered open dispensers') + assert.strictEqual(decoder.captured.length, 1) + assert.strictEqual(decoder.captured[0].destinationAddress, ORACLE_A) + }) + it('captures the oracle of a NON-top-ranked open dispenser above the gate', async () => { const model = new DispenserModel() const decoder = buildDecoder(twoOpenThenRefill(ORACLE_A), model, ABOVE) @@ -281,23 +311,27 @@ describe('DISPENSER PRICE v1 oracle-fee output capture', function () { // The defect itself, pinned. Changing this is a consensus change: a re-decode // of pre-flag-day history must reproduce the output set the fleet wrote live. const model = new DispenserModel() - const decoder = buildDecoder(twoOpenThenRefill(ORACLE_A), model, BELOW) + await withSetCaptureDisarmed(async () => { + const decoder = buildDecoder(twoOpenThenRefill(ORACLE_A), model, BELOW) - await decoder.start() + await decoder.start() - assert.strictEqual(model.rows.length, 2, 'both creates registered open dispensers') - assert.strictEqual(decoder.captured.length, 0, - 'below the gate the wrong oracle is resolved and no output is persisted') + assert.strictEqual(model.rows.length, 2, 'both creates registered open dispensers') + assert.strictEqual(decoder.captured.length, 0, + 'below the gate the wrong oracle is resolved and no output is persisted') + }) }) it('keeps the legacy single-pick below the gate: the top-ranked row still captures', async () => { const model = new DispenserModel() - const decoder = buildDecoder(twoOpenThenRefill(ORACLE_B), model, BELOW) + await withSetCaptureDisarmed(async () => { + const decoder = buildDecoder(twoOpenThenRefill(ORACLE_B), model, BELOW) - await decoder.start() + await decoder.start() - assert.strictEqual(decoder.captured.length, 1) - assert.strictEqual(decoder.captured[0].destinationAddress, ORACLE_B) + assert.strictEqual(decoder.captured.length, 1) + assert.strictEqual(decoder.captured[0].destinationAddress, ORACLE_B) + }) }) }) diff --git a/test/unit/migration-runner.test.js b/test/unit/migration-runner.test.js index fefd624..e2ebaf9 100644 --- a/test/unit/migration-runner.test.js +++ b/test/unit/migration-runner.test.js @@ -517,6 +517,47 @@ describe('runMigrations() checksum re-bless path @regression', function () { assert.deepStrictEqual(updates, []); assert.deepStrictEqual(res, { applied: [], pending: [], baselined: [], lockSkipped: false }); }); + + // a production BTC decoder recorded 2026-05-28-unique-index-tables.sql at its ORIGINAL + // shipped revision (8151979, deployed 2026-06-10 .. 2026-07-10), which predates the + // `@mempool_has_ids` guard revision the table pinned. Only the guard revision was + // blessed, so that node tripped the immutability guard at every startup. Drive the real + // runMigrations() over the real committed file with the historical hash in the ledger: + // it must heal to the committed sha256 rather than throw. + describe('2026-05-28-unique-index-tables.sql historical revisions', function () { + + const REAL_FILE = '2026-05-28-unique-index-tables.sql'; + // sha256 of the file as shipped by 8151979, before the mempool guard landed. This is + // what the affected fleet DBs carry in schema_migrations; it is a fixed historical + // fact, so it is pinned here rather than recomputed. + const SHIPPED_8151979 = 'e1f7df7973881b6fcaa5535fe5aca86b82bb7f45fa4e7e5fdcf9c5859c468207'; + const GUARDED_50a5e83 = '8845b9addc0990b0433f8862969b57cb472535474b4b4d5576c408db777b57ce'; + + const realPath = path.join(__dirname, '..', '..', 'src', 'sql', 'migrations', REAL_FILE); + const realContent = fs.readFileSync(realPath, 'utf8'); + const realSum = crypto.createHash('sha256').update(realContent).digest('hex'); + + for (const [label, recorded] of [ + ['the original shipped revision (8151979)', SHIPPED_8151979], + ['the guarded revision (50a5e83)', GUARDED_50a5e83], + ]) { + it('heals a ledger recording ' + label, async function () { + const root = tmpMigrationsDir(REAL_FILE, realContent); + const { db, updates } = makeDb(root, [{ name: REAL_FILE, checksum: recorded }]); + const res = await db.runMigrations({ includeManual: true }); + assert.deepStrictEqual(updates, [[realSum, REAL_FILE]], + 'expected the ledger to be healed to the committed checksum'); + assert.deepStrictEqual(res.applied, [], 'an already-applied file must not re-run'); + }); + } + + it('still fails closed on a revision that was never shipped', async function () { + const root = tmpMigrationsDir(REAL_FILE, realContent); + const { db, updates } = makeDb(root, [{ name: REAL_FILE, checksum: 'd'.repeat(64) }]); + await assert.rejects(() => db.runMigrations({ includeManual: true }), /content CHANGED/); + assert.deepStrictEqual(updates, [], 'an unpinned hash must not be healed'); + }); + }); }); // Functional coverage of the per-file scoping (--file / opts.only): drive the real diff --git a/test/unit/nodeReachabilityStatus.test.js b/test/unit/nodeReachabilityStatus.test.js new file mode 100644 index 0000000..6d5ce29 --- /dev/null +++ b/test/unit/nodeReachabilityStatus.test.js @@ -0,0 +1,280 @@ +/********************************************************************* + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * A node that never answered, made visible. + * + * An operator ran a decoder whose coin node answered no RPC at all: the log + * carried "Getting timeout trying to get blockchain info, trying again..." 2099 + * times over five and a half days, the restart count stayed 0 and the container + * healthcheck read healthy throughout. Nothing on any surface said the service had + * never reached its node. + * + * The healthy VERDICT is deliberately unchanged: isStalled() reports a + * never-polled decoder as not stale on purpose, because a restart cannot fix an + * upstream outage and gating on it re-opens the autoheal restart flap. What these + * pin is the VISIBILITY: the connector records when the node last answered and + * when it last failed, nodeReachability() reduces those to node_last_ok_at and + * node_unreachable, and both ride every payload that already carries + * node_catching_up. + */ + +'use strict' + +const assert = require('assert') +const fs = require('fs') +const http = require('http') +const path = require('path') +const express = require('express') + +const BlockchainConnector = require('../../src/BlockchainConnector') +const { nodeReachabilityFrom } = BlockchainConnector +const XChainDecoder = require('../../src/XChainDecoder') +const { registerLiveRoute, nodeReachabilityFields } = require('../../src/api') + +const T0 = Date.parse('2026-09-09T12:00:00.000Z') // connector construction +const OK = Date.parse('2026-09-09T12:10:00.000Z') +const FAIL = Date.parse('2026-09-09T12:20:00.000Z') +const NOW = Date.parse('2026-09-09T13:00:00.000Z') + +const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ + +describe('nodeReachabilityFrom() (the reducer both fields are derived from)', function () { + it('reports nothing wrong before any attempt has been made', function () { + const r = nodeReachabilityFrom(T0, 0, 0, NOW) + assert.deepStrictEqual(r, { node_last_ok_at: null, node_unreachable: null }) + }) + + it('reports the last success and no outage while the latest attempt succeeded', function () { + const r = nodeReachabilityFrom(T0, OK, 0, NOW) + assert.strictEqual(r.node_last_ok_at, '2026-09-09T12:10:00.000Z') + assert.strictEqual(r.node_unreachable, null) + }) + + it('dates an outage from the last success when there was one', function () { + const r = nodeReachabilityFrom(T0, OK, FAIL, NOW) + assert.deepStrictEqual(Object.keys(r.node_unreachable).sort(), + ['last_ok_at', 'seconds', 'since']) + assert.strictEqual(r.node_unreachable.since, '2026-09-09T12:10:00.000Z') + assert.strictEqual(r.node_unreachable.last_ok_at, '2026-09-09T12:10:00.000Z') + assert.strictEqual(r.node_unreachable.seconds, 3000) + assert.strictEqual(r.node_last_ok_at, '2026-09-09T12:10:00.000Z') + }) + + it('dates an outage from connector start when the node NEVER answered', function () { + // The reported defect: no success to date the outage from, so the age is the + // life of the connector, and last_ok_at stays null rather than inventing one. + const r = nodeReachabilityFrom(T0, 0, FAIL, NOW) + assert.strictEqual(r.node_last_ok_at, null) + assert.strictEqual(r.node_unreachable.since, '2026-09-09T12:00:00.000Z') + assert.strictEqual(r.node_unreachable.last_ok_at, null) + assert.strictEqual(r.node_unreachable.seconds, 3600) + }) + + it('clears the outage as soon as one attempt succeeds again', function () { + // Failure at FAIL, success after it: the LATEST attempt is what decides. + const later = FAIL + 60000 + const r = nodeReachabilityFrom(T0, later, FAIL, NOW) + assert.strictEqual(r.node_unreachable, null, + 'a recovered node must not stay latched as unreachable') + assert.strictEqual(r.node_last_ok_at, new Date(later).toISOString()) + }) + + it('treats a failure at the same instant as the last success as recovered', function () { + // Strictly-newer, not newer-or-equal: two events in one millisecond must not + // flip a node that is answering into an outage with a zero-second age. + assert.strictEqual(nodeReachabilityFrom(T0, OK, OK, NOW).node_unreachable, null) + }) + + it('floors the age to whole seconds and never publishes a negative one', function () { + assert.strictEqual(nodeReachabilityFrom(T0, 0, FAIL, T0 + 1999).node_unreachable.seconds, 1) + assert.strictEqual(nodeReachabilityFrom(T0, 0, FAIL, T0 - 5000).node_unreachable.seconds, 0, + 'a probe racing the recorded instant must not report a negative outage') + }) + + it('emits ISO instants, not locale strings or epoch numbers', function () { + const r = nodeReachabilityFrom(T0, OK, FAIL, NOW) + assert.match(r.node_last_ok_at, ISO) + assert.match(r.node_unreachable.since, ISO) + assert.strictEqual(new Date(r.node_unreachable.since).toISOString(), r.node_unreachable.since) + }) + + it('defaults `now` to the wall clock, so a caller cannot forget to pass one', function () { + const r = nodeReachabilityFrom(Date.now() - 10000, 0, Date.now()) + assert.ok(r.node_unreachable.seconds >= 9 && r.node_unreachable.seconds <= 11, + 'expected roughly a ten second outage, got ' + r.node_unreachable.seconds) + }) +}) + +describe('the connector records both instants at its single POST choke point', function () { + function newConnector(){ + return new BlockchainConnector('127.0.0.1', '18443', 'u', 'p') + } + + it('starts with never-succeeded, never-failed and a start time', function () { + const c = newConnector() + assert.strictEqual(c.lastNodeOkAt, 0) + assert.strictEqual(c.lastNodeFailAt, 0) + assert.ok(c.startedAt > 0, 'the outage of a node that never answered is dated from here') + assert.deepStrictEqual(c.nodeReachability(), + { node_last_ok_at: null, node_unreachable: null }) + }) + + it('a successful POST stamps lastNodeOkAt and clears the verdict', async function () { + const c = newConnector() + c.lastNodeFailAt = Date.now() - 1000 + // rpcPost is the choke point every RPC method funnels through, so stubbing the + // transport under it exercises the real recording path. + const axios = require('axios') + const realPost = axios.post + axios.post = async () => ({ data: { result: 'ok' } }) + try { + await c.rpcPost({ method: 'getblockchaininfo' }) + } finally { + axios.post = realPost + } + assert.ok(c.lastNodeOkAt > 0) + assert.strictEqual(c.nodeReachability().node_unreachable, null) + }) + + it('a failing POST stamps lastNodeFailAt and rethrows the original error', async function () { + const c = newConnector() + const axios = require('axios') + const realPost = axios.post + const boom = new Error('timeout of 30000ms exceeded') + boom.code = 'ECONNABORTED' + axios.post = async () => { throw boom } + try { + await assert.rejects(() => c.rpcPost({ method: 'getblockchaininfo' }), + (err) => err.code === 'ECONNABORTED') + } finally { + axios.post = realPost + } + assert.ok(c.lastNodeFailAt > 0) + const r = c.nodeReachability() + assert.strictEqual(r.node_last_ok_at, null, 'this node has never answered') + assert.ok(r.node_unreachable, 'the timeout the operator saw 2099 times must show here') + assert.strictEqual(r.node_unreachable.last_ok_at, null) + }) + + it('every RPC method reaches the recording site through rpcPost', function () { + // Source-level: instrumenting per method is how the next added method silently + // escapes the surface. Nothing in this class may POST around the choke point. + const SRC = fs.readFileSync(path.join(__dirname, '..', '..', 'src', 'BlockchainConnector.js'), 'utf8') + const posts = SRC.match(/axios\.post\(/g) || [] + assert.strictEqual(posts.length, 1, 'axios.post must appear only inside rpcPost') + }) +}) + +describe('the reachability fields ride the health payloads', function () { + const API = fs.readFileSync(path.join(__dirname, '..', '..', 'src', 'api.js'), 'utf8') + + function liveApp(decoder, running = true){ + const app = express() + registerLiveRoute(app, decoder, () => running) + return app + } + + function getLive(app){ + return new Promise((resolve, reject) => { + const server = app.listen(0, () => { + http.get({ port: server.address().port, path: '/live' }, (res) => { + let body = '' + res.on('data', (c) => { body += c }) + res.on('end', () => { server.close(); resolve({ status: res.statusCode, body: JSON.parse(body) }) }) + }).on('error', (e) => { server.close(); reject(e) }) + }) + }) + } + + function probeDecoder(connector){ + const decoder = new XChainDecoder( + 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ) + decoder.lastProcessedBlockIndex = 100 + decoder.blockchainInfoLastBlock = 100 + decoder.blockchainInfoLastRefreshAt = Date.now() + decoder.lastAdvanceAt = Date.now() + decoder.lastPollAt = Date.now() + decoder.db = { ping: async () => true } + decoder.connector = connector || { rpcErrors: 0 } + return decoder + } + + it('/live publishes the outage of a node that has never answered', async function () { + const connector = new BlockchainConnector('127.0.0.1', '18443', 'u', 'p') + connector.rpcErrors = 0 + connector.startedAt = Date.now() - 3600000 + connector.lastNodeFailAt = Date.now() + const res = await getLive(liveApp(probeDecoder(connector))) + assert.strictEqual(res.body.node_last_ok_at, null) + assert.ok(res.body.node_unreachable, 'the field the operator had no way to see') + assert.strictEqual(res.body.node_unreachable.last_ok_at, null) + assert.ok(res.body.node_unreachable.seconds >= 3599) + assert.match(res.body.node_unreachable.since, ISO) + }) + + it('/live still answers 200 while the node is unreachable, by design', async function () { + // The visibility is new; the verdict is not. A restart cannot fix an upstream + // outage, so gating here would restart-flap a decoder that is doing its job. + const connector = new BlockchainConnector('127.0.0.1', '18443', 'u', 'p') + connector.rpcErrors = 0 + connector.lastNodeFailAt = Date.now() + const res = await getLive(liveApp(probeDecoder(connector))) + assert.strictEqual(res.status, 200) + assert.strictEqual(res.body.status, 'healthy') + assert.ok(res.body.node_unreachable) + }) + + it('/live publishes both keys as null when the node is answering', async function () { + const connector = new BlockchainConnector('127.0.0.1', '18443', 'u', 'p') + connector.rpcErrors = 0 + connector.lastNodeOkAt = Date.now() + const res = await getLive(liveApp(probeDecoder(connector))) + assert.ok('node_last_ok_at' in res.body, 'an omitted key reads as "this build cannot tell you"') + assert.ok('node_unreachable' in res.body) + assert.match(res.body.node_last_ok_at, ISO) + assert.strictEqual(res.body.node_unreachable, null) + }) + + it('/live carries both keys against a decoder whose connector is an old stub', async function () { + const res = await getLive(liveApp(probeDecoder())) + assert.strictEqual(res.body.node_last_ok_at, null) + assert.strictEqual(res.body.node_unreachable, null) + }) + + // /status and the JSON-RPC health method are built inside startApi(), which binds a + // port and a live decoder, so those two are pinned at source level, the shape + // nodeCatchingUpStatus.test.js uses for the same reason. + it('every payload carrying node_catching_up also spreads the reachability fields', function () { + const sites = [] + for (let at = API.indexOf('node_catching_up:'); at !== -1; at = API.indexOf('node_catching_up:', at + 1)) sites.push(at) + assert.strictEqual(sites.length, 3, 'three payloads carry node_catching_up: /live, rpc health, /status') + + for (const at of sites){ + assert.ok(/\.\.\.nodeReachabilityFields\(decoder\),/.test(API.slice(at, at + 500)), + 'a node_catching_up payload at offset ' + at + ' ships without the reachability fields') + } + }) + + it('reads the fields fail-soft, so a payload built without a connector cannot throw', function () { + assert.deepStrictEqual(nodeReachabilityFields(undefined), + { node_last_ok_at: null, node_unreachable: null }) + assert.deepStrictEqual(nodeReachabilityFields({}), + { node_last_ok_at: null, node_unreachable: null }) + assert.deepStrictEqual(nodeReachabilityFields({ connector: { rpcErrors: 0 } }), + { node_last_ok_at: null, node_unreachable: null }) + assert.deepStrictEqual( + nodeReachabilityFields({ connector: { nodeReachability(){ throw new Error('boom') } } }), + { node_last_ok_at: null, node_unreachable: null }) + }) +}) diff --git a/test/unit/oracleFeeOutputActivationConformance.test.js b/test/unit/oracleFeeOutputActivationConformance.test.js index 84d1068..b855014 100644 --- a/test/unit/oracleFeeOutputActivationConformance.test.js +++ b/test/unit/oracleFeeOutputActivationConformance.test.js @@ -96,10 +96,21 @@ describe('ORACLE_FEE_OUTPUT_ACTIVATION conformance', function () { // membership over every open Mode B dispenser of the paying source. That changes the set of // outputs persisted to transaction_outputs, so it is consensus-affecting in both directions: // arming it early on a fleet that has not deployed forks the chain, and arming it in the past -// rewrites agreed history on a re-decode. null means DISARMED, which is the fail-closed -// default a network sits at until its maintainers ratify an instant. +// rewrites agreed history on a re-decode. mainnet is ARMED by the 2026-09-09 ruling at the base +// gate's own instant, the earliest the ordering below permits, and rewrites nothing because the +// indexed mainnet history holds 0 dispensers (measured 2026-09-09). null stays the fail-closed +// reading for any network that has not armed. describe('ORACLE_FEE_SET_CAPTURE_ACTIVATION conformance', function () { + it('arms mainnet at the base gate instant by the 2026-09-09 ruling', function () { + // Teeth for the ruling AND for its ordering constraint in one place: the widening + // starts exactly where capture itself starts, so no mainnet block sits between the two + // gates, and a re-decode of the (dispenser-free) history persists the same output set. + assert.strictEqual(ORACLE_FEE_SET_CAPTURE_ACTIVATION.mainnet, PINNED_MAINNET_ACTIVATION); + assert.strictEqual(ORACLE_FEE_SET_CAPTURE_ACTIVATION.mainnet, + ORACLE_FEE_OUTPUT_ACTIVATION.mainnet); + }); + it('carries a block time or null (DISARMED) per network, regtest genesis-on', function () { const networks = Object.keys(ORACLE_FEE_SET_CAPTURE_ACTIVATION); assert.deepStrictEqual(networks.sort(), ['mainnet', 'regtest', 'testnet'], diff --git a/test/unit/shutdown.test.js b/test/unit/shutdown.test.js new file mode 100644 index 0000000..ac5acfb --- /dev/null +++ b/test/unit/shutdown.test.js @@ -0,0 +1,265 @@ +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// Pins the container exit path. `docker stop` sends SIGTERM to node (PID 1 via the +// Dockerfile's exec-form CMD) and this drain is everything between that signal and +// the process ending. Before it existed the handler only set stopFlag, the listener +// and the pool kept the process alive, and every stop ended in SIGKILL (exit 137). + +const assert = require('assert'); +const { createShutdown, createDecoderDrain, closeServer, closeDatabases, resolveTimeoutMs, DEFAULT_SHUTDOWN_TIMEOUT_MS } = require('../../src/shutdown'); + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +async function waitUntil(predicate, timeoutMs = 5000, intervalMs = 10){ + const deadline = Date.now() + timeoutMs; + while(Date.now() < deadline){ + if(await predicate()) return true; + await sleep(intervalMs); + } + return Boolean(await predicate()); +} + +const silentLog = { log(){}, warn(){}, error(){} }; + +// Minimal XChainDecoder stand-in: records call ORDER, because the ordering is the +// contract (health flag before stop, pools closed last). +function makeDecoder(order){ + let resolveLoop; + const loop = new Promise((res) => { resolveLoop = res; }); + const db = (name) => ({ + closed: false, + async close(){ this.closed = true; order.push('close:' + name); } + }); + return { + stopped: false, + db: db('db'), + mempoolDb: db('mempoolDb'), + loop, + // The real stop() only sets stopFlag; the loop breaks at the top of its next + // iteration, which the test models by resolving the loop promise later. + stop(){ this.stopped = true; order.push('stop'); setImmediate(resolveLoop); } + }; +} + +function makeServer(order){ + return { + closed: false, + idleDropped: false, + close(cb){ this.closed = true; order.push('server.close'); setImmediate(cb); }, + closeIdleConnections(){ this.idleDropped = true; } + }; +} + +describe('graceful shutdown', function(){ + + describe('createShutdown', function(){ + + it('runs the drain and exits zero when it completes', async function(){ + const codes = []; + let drained = false; + const shutdown = createShutdown({ + drain: async () => { drained = true; }, + exit: (c) => codes.push(c), + log: silentLog + }); + shutdown('SIGTERM'); + assert.ok(await waitUntil(() => codes.length > 0), 'timed out waiting for the clean drain to reach exit()'); + assert.strictEqual(drained, true); + assert.deepStrictEqual(codes, [0]); + }); + + it('is idempotent: a second signal does not re-enter the drain', async function(){ + const codes = []; + let calls = 0; + const shutdown = createShutdown({ + drain: async () => { calls++; await sleep(20); }, + exit: (c) => codes.push(c), + log: silentLog + }); + shutdown('SIGTERM'); + shutdown('SIGTERM'); + shutdown('SIGINT'); + assert.ok(await waitUntil(() => codes.length > 0), 'timed out waiting for the single in-flight drain to reach exit()'); + assert.strictEqual(calls, 1, 'drain must run exactly once'); + assert.deepStrictEqual(codes, [0]); + }); + + // The reason the handler is safe to install at all: registering one REMOVES + // node's default terminate, so without this bound a hung drain turns every + // stop into a container that lingers until the supervisor's grace expires. + it('hard-exits non-zero when the drain overruns its budget', async function(){ + const codes = []; + const shutdown = createShutdown({ + drain: () => new Promise(() => {}), // never settles + timeoutMs: 20, + exit: (c) => codes.push(c), + log: silentLog + }); + shutdown('SIGTERM'); + assert.ok(await waitUntil(() => codes.length > 0), 'timed out waiting for the hard-exit timer to fire'); + assert.deepStrictEqual(codes, [1]); + }); + + it('exits non-zero when the drain throws, and only once', async function(){ + const codes = []; + const shutdown = createShutdown({ + drain: async () => { throw new Error('pool refused to close'); }, + timeoutMs: 50, + exit: (c) => codes.push(c), + log: silentLog + }); + shutdown('SIGTERM'); + // Outlive the 50ms hard-exit timer to prove it was cleared. + await sleep(120); + assert.deepStrictEqual(codes, [1]); + }); + + it('does not fire the hard-exit timer after a clean drain', async function(){ + const codes = []; + const shutdown = createShutdown({ + drain: async () => {}, + timeoutMs: 20, + exit: (c) => codes.push(c), + log: silentLog + }); + shutdown('SIGTERM'); + await sleep(80); + assert.deepStrictEqual(codes, [0], 'a cleared timer must not add a second exit'); + }); + }); + + describe('resolveTimeoutMs', function(){ + it('prefers an explicit budget, then the env var, then the default', function(){ + assert.strictEqual(resolveTimeoutMs(1234, {}), 1234); + assert.strictEqual(resolveTimeoutMs(undefined, { SHUTDOWN_TIMEOUT_MS: '4321' }), 4321); + assert.strictEqual(resolveTimeoutMs(undefined, {}), DEFAULT_SHUTDOWN_TIMEOUT_MS); + assert.strictEqual(resolveTimeoutMs(0, { SHUTDOWN_TIMEOUT_MS: 'nonsense' }), DEFAULT_SHUTDOWN_TIMEOUT_MS); + }); + + // xchain-node stops a decoder with a 120 s budget and stamps it on the + // container; the drain's own bound must end in a LOGGED exit before that. + it('stays under the 120 s budget xchain-node gives a decoder', function(){ + assert.ok(DEFAULT_SHUTDOWN_TIMEOUT_MS < 120000, + 'a budget at or above the container stop-timeout ends in the daemon\'s SIGKILL, which is what this replaces'); + assert.ok(DEFAULT_SHUTDOWN_TIMEOUT_MS > 10000, + 'a block boundary on a mainnet chain is not reached in docker\'s ten seconds'); + }); + }); + + describe('closeServer', function(){ + it('resolves once, and drops idle keep-alive sockets that would hold close() open', async function(){ + const order = []; + const server = makeServer(order); + await closeServer(server); + assert.strictEqual(server.closed, true); + assert.strictEqual(server.idleDropped, true); + }); + + it('resolves on a missing or closeless server rather than hanging the drain', async function(){ + await closeServer(null); + await closeServer({}); + }); + }); + + describe('closeDatabases', function(){ + it('closes each handle once and survives one that refuses', async function(){ + let closes = 0; + const ok = { async close(){ closes++; } }; + const bad = { async close(){ throw new Error('refused'); } }; + await closeDatabases([ok, ok, bad, null, {}], silentLog); + assert.strictEqual(closes, 1); + }); + }); + + describe('createDecoderDrain', function(){ + + it('flips health, stops the decoder, drains the server and loop, then closes both pools', async function(){ + const order = []; + const decoder = makeDecoder(order); + const server = makeServer(order); + let running = true; + + const drain = createDecoderDrain({ + decoder, + server, + loopSettled: decoder.loop, + onDraining: () => { running = false; order.push('health-flag'); }, + log: silentLog + }); + await drain(); + + assert.strictEqual(running, false, '/live must stop reporting the decoder running'); + assert.strictEqual(decoder.stopped, true); + assert.strictEqual(server.closed, true); + assert.ok(order.indexOf('health-flag') < order.indexOf('stop'), 'health flag must flip before stop(), not after'); + for(const name of ['db', 'mempoolDb']){ + assert.ok(order.indexOf('close:' + name) > order.indexOf('server.close'), name + ' must close after the server has drained'); + assert.ok(order.indexOf('close:' + name) > order.indexOf('stop'), name + ' must close after the parse loop was told to stop'); + } + assert.ok(decoder.db.closed && decoder.mempoolDb.closed); + }); + + it('waits for the parse loop to break before closing pools', async function(){ + const order = []; + const decoder = makeDecoder(order); + const server = makeServer(order); + + let breakLoop; + const loop = new Promise((res) => { breakLoop = res; }); + const drain = createDecoderDrain({ decoder, server, loopSettled: loop, log: silentLog }); + + let settled = false; + const running = drain().then(() => { settled = true; }); + + await sleep(30); + assert.strictEqual(settled, false, 'the drain must not finish while the parse loop is mid-block'); + assert.strictEqual(decoder.db.closed, false, 'closing a pool under an open block transaction is the exact abort this fix removes'); + + breakLoop(); + await running; + assert.strictEqual(decoder.db.closed, true); + }); + + it('survives a rejected loop promise', async function(){ + const order = []; + const decoder = makeDecoder(order); + const server = makeServer(order); + const drain = createDecoderDrain({ + decoder, server, + loopSettled: Promise.reject(new Error('fatal decoder error')), + log: silentLog + }); + await drain(); + assert.strictEqual(decoder.db.closed, true); + }); + + it('drains a partially-built process without throwing', async function(){ + const drain = createDecoderDrain({ decoder: null, server: null, log: silentLog }); + await drain(); + }); + }); + + // The Database class had no close() at all, which is half of why the process + // could not exit: the pool's sockets kept the event loop alive. + describe('Database.close()', function(){ + it('ends the pool once and releases a held transaction connection first', async function(){ + const Database = require('../../src/db'); + const db = Object.create(Database.prototype); + const calls = []; + db.transactionConnection = { async release(){ calls.push('release'); } }; + db.pool = { async end(){ calls.push('end'); } }; + await db.close(); + await db.close(); + assert.deepStrictEqual(calls, ['release', 'end']); + assert.strictEqual(db.transactionConnection, null); + assert.strictEqual(db.pool, null); + }); + }); +});