From 4ff1dc8a347b7921d29e8d84e1315aff36b81c72 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:27:19 -0700 Subject: [PATCH 1/2] node: more /metrics series and NixOS health options Operators need tip age, difficulty, peer network, byte totals, and mempool pressure on the same scrape, each taken from a value RPC already publishes. services.rbitcoin.health.enable passes --health-listen, and metrics adds --metrics plus a Prometheus scrape job when Prometheus is enabled. --- changelog.d/health-metrics-nixos.md | 10 +++ crates/rbitcoin-consensus/src/header.rs | 21 +++++ crates/rbitcoin-consensus/src/lib.rs | 2 +- crates/rbitcoin-net/src/lib.rs | 6 +- crates/rbitcoin-net/src/peers.rs | 15 ++++ crates/rbitcoin-node/src/health/metrics.rs | 95 +++++++++++++++++++-- crates/rbitcoin-rpc/src/methods/chain.rs | 16 +--- crates/rbitcoin-rpc/src/methods/net.rs | 11 +-- crates/rbitcoin-test/tests/cross_surface.rs | 49 ++++++++++- docs/operator/operations.md | 17 +++- docs/operator/setup.md | 5 ++ nix/modules/rbitcoin.nix | 45 +++++++++- nix/tests/nixos-module-eval.nix | 50 +++++++++++ nix/tests/nixos-module-runtime.nix | 5 ++ 14 files changed, 307 insertions(+), 40 deletions(-) create mode 100644 changelog.d/health-metrics-nixos.md diff --git a/changelog.d/health-metrics-nixos.md b/changelog.d/health-metrics-nixos.md new file mode 100644 index 000000000..0faf16125 --- /dev/null +++ b/changelog.d/health-metrics-nixos.md @@ -0,0 +1,10 @@ +Added + +- **More Prometheus gauges.** `/metrics` also exposes verification progress, + tip age, difficulty, peer counts by network, outbound time offset, P2P + byte totals, and mempool min fee, weight cap, orphans, and unbroadcast + count. Each one is a value RPC already publishes. +- **NixOS health and metrics.** `services.rbitcoin.health.enable` passes + `--health-listen` (default `127.0.0.1:9332`). `services.rbitcoin.metrics` + passes `--metrics` and adds a Prometheus scrape job when + `services.prometheus.enable` is set. diff --git a/crates/rbitcoin-consensus/src/header.rs b/crates/rbitcoin-consensus/src/header.rs index 37574a558..aee9d417c 100644 --- a/crates/rbitcoin-consensus/src/header.rs +++ b/crates/rbitcoin-consensus/src/header.rs @@ -304,6 +304,22 @@ fn min_diff_bits( Some(bits) } +/// Compact `nBits` as a difficulty relative to the max target (Core `GetDifficulty`). +pub fn difficulty_from_bits(bits: u32) -> f64 { + let n_shift = ((bits >> 24) & 0xff) as i32; + let mut ddiff = (0x0000_ffff_u64 as f64) / ((bits & 0x00ff_ffff) as f64); + let mut shift = n_shift - 29; + while shift < 0 { + ddiff *= 256.0; + shift += 1; + } + while shift > 0 { + ddiff /= 256.0; + shift -= 1; + } + ddiff +} + fn header_bits_at(query: &Query, height: Height) -> Result { if let Some((_fk, rec)) = query.header_at_height(height)? { return Ok(rec.bits); @@ -324,6 +340,11 @@ mod median_time_past_tests { use rbitcoin_store::{HeaderRecord, InputRecord, OutputRecord, TxRecord}; use std::time::{SystemTime, UNIX_EPOCH}; + #[test] + fn max_target_bits_are_difficulty_one() { + assert_eq!(difficulty_from_bits(0x1d00_ffff), 1.0); + } + #[test] fn pow_hash_meets_target_meet_miss_and_limit() { let rt = ChainParams::regtest(); diff --git a/crates/rbitcoin-consensus/src/lib.rs b/crates/rbitcoin-consensus/src/lib.rs index 316565891..09a2d9431 100644 --- a/crates/rbitcoin-consensus/src/lib.rs +++ b/crates/rbitcoin-consensus/src/lib.rs @@ -70,7 +70,7 @@ pub use convert::header_to_record; pub(crate) use convert::{block_to_apply, block_to_apply_with_txids_prev}; pub use error::{block_reject_log_line, block_reject_reason, script_flag_paren, ConsensusError}; pub use header::{ - expected_next_bits, median_time_past, next_work_bits, validate_header, + difficulty_from_bits, expected_next_bits, median_time_past, next_work_bits, validate_header, validate_header_on_parent, }; pub use index_writebehind::{ diff --git a/crates/rbitcoin-net/src/lib.rs b/crates/rbitcoin-net/src/lib.rs index 384ec78b7..ea4d44f77 100644 --- a/crates/rbitcoin-net/src/lib.rs +++ b/crates/rbitcoin-net/src/lib.rs @@ -60,9 +60,9 @@ pub use peer::{ }; pub use peer_dos::DEFAULT_MAX_INBOUND; pub use peers::{ - connection_counts, parse_peer_addr, parse_peer_addr_with_port, parse_peer_net, - pick_stale_follow_evict, DialRequest, DialTarget, LivePeer, PeerConnType, PeerHub, PeerInfo, - PeerOut, PingAction, + connection_counts, outbound_time_offset, parse_peer_addr, parse_peer_addr_with_port, + parse_peer_net, pick_stale_follow_evict, DialRequest, DialTarget, LivePeer, PeerConnType, + PeerHub, PeerInfo, PeerOut, PingAction, }; pub use perf_meter::RequestMeter; pub(crate) use rbitcoin_mempool::MempoolGraphStats; diff --git a/crates/rbitcoin-net/src/peers.rs b/crates/rbitcoin-net/src/peers.rs index f36d3ede8..592636c75 100644 --- a/crates/rbitcoin-net/src/peers.rs +++ b/crates/rbitcoin-net/src/peers.rs @@ -1188,6 +1188,21 @@ pub fn connection_counts(peers: &[PeerInfo]) -> (u64, u64) { (inbound as u64, (peers.len() - inbound) as u64) } +/// `getnetworkinfo.timeoffset`: median VERSION offset of completed outbound peers. +/// Even counts use the upper middle. Inbound-only or empty is 0. +pub fn outbound_time_offset(peers: &[PeerInfo]) -> i64 { + let mut offs: Vec = peers + .iter() + .filter(|p| !p.inbound && p.handshake_complete) + .map(|p| p.time_offset_secs) + .collect(); + if offs.is_empty() { + return 0; + } + offs.sort_unstable(); + offs[offs.len() / 2] +} + /// RPC-facing snapshot. #[derive(Clone, Debug)] pub struct PeerInfo { diff --git a/crates/rbitcoin-node/src/health/metrics.rs b/crates/rbitcoin-node/src/health/metrics.rs index f6879beaa..5b7d31214 100644 --- a/crates/rbitcoin-node/src/health/metrics.rs +++ b/crates/rbitcoin-node/src/health/metrics.rs @@ -12,7 +12,8 @@ pub(super) const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8" /// One scrape. Chain reads may touch the store and the mempool totals take /// its lock, so call from the blocking pool. Cost: those chain reads /// (`best_header_height`, tip header, `in_ibd`, scripthash lag), one peer -/// snapshot, and one mempool fold (the same fold as `getmempoolinfo`). +/// snapshot plus `byte_totals`, and one mempool fold (the same fold as +/// `getmempoolinfo`) plus the min-fee, cap, orphan, and unbroadcast reads. pub(super) fn render(status: &NodeStatus) -> String { let mut out = Exposition::default(); out.family( @@ -67,7 +68,8 @@ pub(super) fn render(status: &NodeStatus) -> String { chain_gauges(&mut out, chain, status.sh_index); } if let Some(peers) = status.peers.get() { - let (inbound, outbound) = rbitcoin_net::connection_counts(&peers.snapshot()); + let rows = peers.snapshot(); + let (inbound, outbound) = rbitcoin_net::connection_counts(&rows); out.family( "rbitcoin_connections", "gauge", @@ -75,6 +77,34 @@ pub(super) fn render(status: &NodeStatus) -> String { ); out.sample("rbitcoin_connections", "{direction=\"in\"}", inbound); out.sample("rbitcoin_connections", "{direction=\"out\"}", outbound); + out.gauge( + "rbitcoin_peer_time_offset_seconds", + "Median outbound peer clock offset (getnetworkinfo.timeoffset).", + rbitcoin_net::outbound_time_offset(&rows), + ); + out.family( + "rbitcoin_peers", + "gauge", + "Peer connections by network (getpeerinfo.network).", + ); + for network in ["ipv4", "ipv6", "onion", "i2p", "cjdns"] { + let n = rows + .iter() + .filter(|p| p.net.network_label() == network) + .count(); + out.sample("rbitcoin_peers", &format!("{{network=\"{network}\"}}"), n); + } + let (recv, sent) = peers.byte_totals(); + out.counter( + "rbitcoin_network_receive_bytes_total", + "P2P bytes received (getnettotals.totalbytesrecv).", + recv, + ); + out.counter( + "rbitcoin_network_transmit_bytes_total", + "P2P bytes sent (getnettotals.totalbytessent).", + sent, + ); } if let Some(mempool) = status.mempool.get() { let (size, vbytes, _fee) = mempool.live_adjusted_totals(); @@ -88,6 +118,27 @@ pub(super) fn render(status: &NodeStatus) -> String { "Sum of mempool virtual sizes (getmempoolinfo.bytes).", vbytes, ); + out.gauge( + "rbitcoin_mempool_min_fee_sat_per_kvb", + "Mempool min fee in sat/kvB (getmempoolinfo.mempoolminfee).", + mempool.mempool_min_fee_sat_kvb(), + ); + out.gauge( + "rbitcoin_mempool_max_weight", + "Mempool weight cap in weight units (getmempoolinfo.maxmempool).", + mempool.max_weight(), + ); + let (orphans, _orphan_wu) = mempool.orphan_stats(); + out.gauge( + "rbitcoin_mempool_orphan_transactions", + "Orphan transactions (getmempoolinfo.orphanage.size).", + orphans, + ); + out.gauge( + "rbitcoin_mempool_unbroadcast_transactions", + "Local transactions not yet requested (getmempoolinfo.unbroadcastcount).", + mempool.unbroadcast_count(), + ); let (accepts, rejects) = mempool.accept_totals(); out.counter( "rbitcoin_mempool_accepts_total", @@ -142,24 +193,54 @@ fn seconds(us: u64) -> f64 { fn chain_gauges(out: &mut Exposition, chain: &ChainHub, sh_index: bool) { let tip = chain.query.tip_height(); + let blocks = tip.map_or(0, |h| h.0); out.gauge( "rbitcoin_blocks", "Active chain height (getblockchaininfo.blocks).", - tip.map_or(0, |h| h.0), + blocks, ); + let headers = chain.best_header_height(); out.gauge( "rbitcoin_headers", "Best header height (getblockchaininfo.headers).", - chain.best_header_height(), + headers, ); - let time = tip - .and_then(|h| chain.query.header_at_height(h).ok().flatten()) - .map_or(0, |(_, rec)| rec.timestamp); + let progress = if headers == 0 { + 1.0 + } else { + (f64::from(blocks) / f64::from(headers)).clamp(0.0, 1.0) + }; + out.gauge( + "rbitcoin_verification_progress", + "blocks/headers (getblockchaininfo.verificationprogress).", + progress, + ); + let rec = tip.and_then(|h| chain.query.header_at_height(h).ok().flatten()); + let time = rec.as_ref().map_or(0, |(_, rec)| rec.timestamp); out.gauge( "rbitcoin_tip_time_seconds", "Tip block time (getblockchaininfo.time).", time, ); + let age = rec.as_ref().map_or(0, |(_, rec)| { + chain + .clock + .now_secs() + .saturating_sub(u64::from(rec.timestamp)) + }); + out.gauge( + "rbitcoin_tip_age_seconds", + "Seconds since the tip block time.", + age, + ); + let difficulty = rec.map_or(0.0, |(_, rec)| { + rbitcoin_consensus::difficulty_from_bits(rec.bits) + }); + out.gauge( + "rbitcoin_difficulty", + "Tip difficulty (getblockchaininfo.difficulty).", + difficulty, + ); out.gauge( "rbitcoin_initial_block_download", "1 during initial block download (getblockchaininfo.initialblockdownload).", diff --git a/crates/rbitcoin-rpc/src/methods/chain.rs b/crates/rbitcoin-rpc/src/methods/chain.rs index 822f1a849..12986b366 100644 --- a/crates/rbitcoin-rpc/src/methods/chain.rs +++ b/crates/rbitcoin-rpc/src/methods/chain.rs @@ -133,21 +133,7 @@ pub(crate) fn rpc_warnings(ctx: &RpcContext) -> Vec { w } -pub(crate) fn difficulty_from_bits(bits: u32) -> f64 { - // Compact target → difficulty relative to max target (same class as Core). - let n_shift = ((bits >> 24) & 0xff) as i32; - let mut ddiff = (0x0000_ffff_u64 as f64) / ((bits & 0x00ff_ffff) as f64); - let mut shift = n_shift - 29; - while shift < 0 { - ddiff *= 256.0; - shift += 1; - } - while shift > 0 { - ddiff /= 256.0; - shift -= 1; - } - ddiff -} +pub(crate) use rbitcoin_consensus::difficulty_from_bits; pub(crate) fn difficulty_at_tip(ctx: &RpcContext) -> Result { let tip = ctx diff --git a/crates/rbitcoin-rpc/src/methods/net.rs b/crates/rbitcoin-rpc/src/methods/net.rs index 8560a1bfb..6034a3532 100644 --- a/crates/rbitcoin-rpc/src/methods/net.rs +++ b/crates/rbitcoin-rpc/src/methods/net.rs @@ -64,16 +64,7 @@ fn peer_block_connected(ctx: &RpcContext, hash: &bitcoin::BlockHash) -> bool { } fn outbound_median_time_offset(rows: &[rbitcoin_net::PeerInfo]) -> i64 { - let mut offs: Vec = rows - .iter() - .filter(|p| !p.inbound && p.handshake_complete) - .map(|p| p.time_offset_secs) - .collect(); - if offs.is_empty() { - return 0; - } - offs.sort_unstable(); - offs[offs.len() / 2] + rbitcoin_net::outbound_time_offset(rows) } pub(crate) fn peerinfo_json(ctx: &RpcContext, p: rbitcoin_net::PeerInfo) -> Value { diff --git a/crates/rbitcoin-test/tests/cross_surface.rs b/crates/rbitcoin-test/tests/cross_surface.rs index 50f3d740b..1ff3c6d68 100644 --- a/crates/rbitcoin-test/tests/cross_surface.rs +++ b/crates/rbitcoin-test/tests/cross_surface.rs @@ -220,6 +220,8 @@ async fn pin_metrics_equal_rpc( let chain = jsonrpc(rpc_addr, "getblockchaininfo", json!([])).await["result"].clone(); let net = jsonrpc(rpc_addr, "getnetworkinfo", json!([])).await["result"].clone(); let mempool = jsonrpc(rpc_addr, "getmempoolinfo", json!([])).await["result"].clone(); + let peers = jsonrpc(rpc_addr, "getpeerinfo", json!([])).await["result"].clone(); + let totals = jsonrpc(rpc_addr, "getnettotals", json!([])).await["result"].clone(); let m = scrape_metrics(health_addr).await; let num = |v: &Value| v.as_f64().unwrap_or_else(|| panic!("number: {v}")); let flag = |b: bool| if b { 1.0 } else { 0.0 }; @@ -249,14 +251,59 @@ async fn pin_metrics_equal_rpc( ), ("rbitcoin_mempool_transactions", num(&mempool["size"])), ("rbitcoin_mempool_bytes", num(&mempool["bytes"])), + ( + "rbitcoin_verification_progress", + num(&chain["verificationprogress"]), + ), + ("rbitcoin_difficulty", num(&chain["difficulty"])), + ("rbitcoin_peer_time_offset_seconds", num(&net["timeoffset"])), + ("rbitcoin_mempool_max_weight", num(&mempool["maxmempool"])), + ( + "rbitcoin_mempool_orphan_transactions", + num(&mempool["orphanage"]["size"]), + ), + ( + "rbitcoin_mempool_unbroadcast_transactions", + num(&mempool["unbroadcastcount"]), + ), ] { assert_eq!(m.get(series), Some(&want), "{series}: {m:?}"); } - assert!(m["rbitcoin_scripthash_lag_blocks"] <= 6.0, "{m:?}"); + let min_fee_sat = (num(&mempool["mempoolminfee"]) * 100_000_000.0).round(); + assert_eq!( + m["rbitcoin_mempool_min_fee_sat_per_kvb"], min_fee_sat, + "mempoolminfee: {mempool}" + ); let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_secs_f64(); + let tip_age = now - num(&chain["time"]); + assert!( + (m["rbitcoin_tip_age_seconds"] - tip_age).abs() < 30.0, + "tip age {} vs {tip_age}", + m["rbitcoin_tip_age_seconds"] + ); + let peer_rows = peers.as_array().expect("getpeerinfo array"); + for network in ["ipv4", "ipv6", "onion", "i2p", "cjdns"] { + let want = peer_rows.iter().filter(|p| p["network"] == network).count() as f64; + let series = format!("rbitcoin_peers{{network=\"{network}\"}}"); + assert_eq!(m.get(&series), Some(&want), "{series}: {m:?}"); + } + // Bytes can move between the RPC read and the scrape. Both are the same counters. + let recv = num(&totals["totalbytesrecv"]); + let sent = num(&totals["totalbytessent"]); + assert!( + (m["rbitcoin_network_receive_bytes_total"] - recv).abs() < 1_000_000.0, + "recv {} vs {recv}", + m["rbitcoin_network_receive_bytes_total"] + ); + assert!( + (m["rbitcoin_network_transmit_bytes_total"] - sent).abs() < 1_000_000.0, + "sent {} vs {sent}", + m["rbitcoin_network_transmit_bytes_total"] + ); + assert!(m["rbitcoin_scripthash_lag_blocks"] <= 6.0, "{m:?}"); let started = m["process_start_time_seconds"]; assert!(started <= now && started > now - 600.0, "{m:?}"); #[cfg(any(target_os = "linux", target_os = "macos"))] diff --git a/docs/operator/operations.md b/docs/operator/operations.md index 6c765685a..805668964 100644 --- a/docs/operator/operations.md +++ b/docs/operator/operations.md @@ -407,8 +407,11 @@ which. Counters are the process-lifetime totals behind the 5s DEBUG `tip: perf` line; that line still prints only the change since its previous sample. A scrape runs on the blocking pool: chain reads (`best_header_height`, the tip header, `in_ibd`, and scripthash lag with -`--sh-index`), one peer snapshot, and one mempool fold for -`rbitcoin_mempool_bytes` (the same fold as `getmempoolinfo`). +`--sh-index`), one peer snapshot and a byte-total walk, and one mempool +fold for `rbitcoin_mempool_bytes` (the same fold as `getmempoolinfo`) plus +the min-fee, weight cap, orphan count, and unbroadcast count. NixOS: +`services.rbitcoin.health.enable` and `services.rbitcoin.metrics` +([setup](setup.md#nixos-service)). | Metric | Type | Equals | |--------|------|--------| @@ -419,9 +422,19 @@ sample. A scrape runs on the blocking pool: chain reads | `rbitcoin_headers` | gauge | `getblockchaininfo.headers` | | `rbitcoin_tip_time_seconds` | gauge | `getblockchaininfo.time` | | `rbitcoin_initial_block_download` | gauge | `getblockchaininfo.initialblockdownload` | +| `rbitcoin_verification_progress` | gauge | `getblockchaininfo.verificationprogress` (`blocks/headers`) | +| `rbitcoin_tip_age_seconds` | gauge | Seconds since `getblockchaininfo.time` | +| `rbitcoin_difficulty` | gauge | `getblockchaininfo.difficulty` | | `rbitcoin_connections{direction="in"\|"out"}` | gauge | `getnetworkinfo.connections_in` / `connections_out` | +| `rbitcoin_peers{network}` | gauge | Count of `getpeerinfo` rows for `ipv4`, `ipv6`, `onion`, `i2p`, `cjdns` | +| `rbitcoin_peer_time_offset_seconds` | gauge | `getnetworkinfo.timeoffset` | +| `rbitcoin_network_receive_bytes_total` / `_transmit_bytes_total` | counter | `getnettotals.totalbytesrecv` / `totalbytessent` | | `rbitcoin_mempool_transactions` | gauge | `getmempoolinfo.size` | | `rbitcoin_mempool_bytes` | gauge | `getmempoolinfo.bytes` (virtual size) | +| `rbitcoin_mempool_min_fee_sat_per_kvb` | gauge | `getmempoolinfo.mempoolminfee` in sat/kvB | +| `rbitcoin_mempool_max_weight` | gauge | `getmempoolinfo.maxmempool` (weight units) | +| `rbitcoin_mempool_orphan_transactions` | gauge | `getmempoolinfo.orphanage.size` | +| `rbitcoin_mempool_unbroadcast_transactions` | gauge | `getmempoolinfo.unbroadcastcount` | | `rbitcoin_scripthash_lag_blocks` | gauge | `tip: accept sh_lag=` (with `--sh-index`) | | `rbitcoin_esplora_requests_total` / `_request_seconds_total` | counter | Lifetime sum of `tip: perf esplora req=`, and of that handler's wall time in seconds. The DEBUG line is the last ~5s window (`req=`, `avg_us` in microseconds) | | `rbitcoin_electrum_requests_total` / `_request_seconds_total` | counter | Lifetime sum of `tip: perf electrum req=`, and of that handler's wall time in seconds. The DEBUG line is the last ~5s window (`req=`, `avg_us` in microseconds) | diff --git a/docs/operator/setup.md b/docs/operator/setup.md index 85705d82a..2c3775c10 100644 --- a/docs/operator/setup.md +++ b/docs/operator/setup.md @@ -89,6 +89,11 @@ enables the required scripthash index. `p2p.openFirewall`, JSON-RPC has no firewall option; expose it only through an explicitly managed firewall or tunnel. +`health.enable` binds `--health-listen` (default `127.0.0.1:9332`) for +`/healthz` and `/readyz`. `metrics` adds `--metrics` on that listener and, +when `services.prometheus.enable` is set, a scrape job for the health +address. Leave `health.openFirewall` off: the listener is unauthenticated. + The daemon does not terminate TLS. Keep its application listeners on loopback and compose them with a proxy. This example serves Esplora and RPC over HTTPS, and Electrum as TLS-wrapped TCP on port 50002, using one ACME certificate: diff --git a/nix/modules/rbitcoin.nix b/nix/modules/rbitcoin.nix index bbc924dfb..9180e4d7a 100644 --- a/nix/modules/rbitcoin.nix +++ b/nix/modules/rbitcoin.nix @@ -86,6 +86,9 @@ let ++ optional cfg.electrum.enable (socket cfg.electrum.address cfg.electrum.port) ++ optional cfg.esplora.enable "--esplora-listen" ++ optional cfg.esplora.enable (socket cfg.esplora.address cfg.esplora.port) + ++ optional cfg.health.enable "--health-listen" + ++ optional cfg.health.enable (socket cfg.health.address cfg.health.port) + ++ optional cfg.metrics "--metrics" ++ optional (cfg.scripthashIndex || cfg.electrum.enable || cfg.esplora.enable) "--sh-index" ++ optional cfg.silentPaymentIndex "--sp-tweaks" ++ optional (cfg.proxy != null) "--proxy" @@ -314,6 +317,30 @@ in }; }; + health = { + enable = mkEnableOption "the loopback health listener (/healthz and /readyz)"; + + address = mkOption { + type = types.str; + default = "127.0.0.1"; + description = "Address for --health-listen. Keep this on loopback; it is unauthenticated."; + }; + + port = mkOption { + type = types.port; + default = 9332; + description = "Health listen port. Same default as a bare --health-listen."; + }; + + openFirewall = mkOption { + type = types.bool; + default = false; + description = "Open the health port in the firewall. Leave this off unless a probe must dial a non-loopback address."; + }; + }; + + metrics = mkEnableOption "Prometheus GET /metrics on the health listener"; + rpc = { enable = mkEnableOption "the JSON-RPC listener"; @@ -437,6 +464,10 @@ in assertion = cfg.rpc.cookieFile == null || cfg.rpc.enable; message = "services.rbitcoin.rpc.cookieFile requires rpc.enable (the cookie is accepted on TCP only)"; } + { + assertion = !cfg.metrics || cfg.health.enable; + message = "services.rbitcoin.metrics requires health.enable (--metrics needs --health-listen)"; + } ]; users.groups.${cfg.group} = { }; @@ -512,7 +543,19 @@ in networking.firewall.allowedTCPPorts = optional (cfg.p2p.openFirewall && cfg.p2p.listen) cfg.p2p.port ++ optional (cfg.electrum.enable && cfg.electrum.openFirewall) cfg.electrum.port - ++ optional (cfg.esplora.enable && cfg.esplora.openFirewall) cfg.esplora.port; + ++ optional (cfg.esplora.enable && cfg.esplora.openFirewall) cfg.esplora.port + ++ optional (cfg.health.enable && cfg.health.openFirewall) cfg.health.port; + + services.prometheus.scrapeConfigs = mkIf (cfg.metrics && config.services.prometheus.enable) [ + { + job_name = "rbitcoin"; + static_configs = [ + { + targets = [ (socket cfg.health.address cfg.health.port) ]; + } + ]; + } + ]; environment.systemPackages = [ cfg.package ]; }; diff --git a/nix/tests/nixos-module-eval.nix b/nix/tests/nixos-module-eval.nix index 559706f5c..78c443b9e 100644 --- a/nix/tests/nixos-module-eval.nix +++ b/nix/tests/nixos-module-eval.nix @@ -66,6 +66,8 @@ let openFirewall = true; hiddenService = true; }; + health.enable = true; + metrics = true; }; } ]; @@ -132,6 +134,38 @@ let cookieDataDirMode = cookieInDataDir.config.systemd.tmpfiles.settings."10-rbitcoin"."/var/lib/rbitcoin".d.mode; cookieTcpOnly = "services.rbitcoin.rpc.cookieFile requires rpc.enable (the cookie is accepted on TCP only)"; + metricsNeedHealth = "services.rbitcoin.metrics requires health.enable (--metrics needs --health-listen)"; + metricsWithoutHealth = nixpkgs.lib.nixosSystem { + inherit (pkgs.stdenv.hostPlatform) system; + modules = [ + module + { + services.rbitcoin = { + enable = true; + package = fakePackage; + metrics = true; + }; + } + ]; + }; + prometheus = nixpkgs.lib.nixosSystem { + inherit (pkgs.stdenv.hostPlatform) system; + modules = [ + module + { + services.prometheus.enable = true; + services.rbitcoin = { + enable = true; + package = fakePackage; + health.enable = true; + metrics = true; + }; + } + ]; + }; + rbitcoinScrape = builtins.head ( + builtins.filter (j: j.job_name == "rbitcoin") prometheus.config.services.prometheus.scrapeConfigs + ); failedAssertions = sys: map (a: a.message) (builtins.filter (a: !a.assertion) sys.config.assertions); in @@ -156,6 +190,11 @@ assert defaultCfg.esplora.hiddenService == false; assert defaultCfg.i2p.sam == null; assert defaultCfg.i2p.acceptIncoming == false; assert defaultCfg.cjdns.reachable == false; +assert defaultCfg.health.enable == false; +assert defaultCfg.health.address == "127.0.0.1"; +assert defaultCfg.health.port == 9332; +assert defaultCfg.health.openFirewall == false; +assert defaultCfg.metrics == false; assert cfg.services.rbitcoin.p2p.port == 18444; assert cfg.services.rbitcoin.rpc.port == 18443; assert @@ -198,6 +237,14 @@ assert builtins.match ".*--i2p-accept-incoming.*" execStart != null; assert builtins.match ".*--listen-onion.*" execStart != null; assert builtins.match ".*--cjdns-reachable.*" execStart != null; assert builtins.match ".*--prune-seqsigwit.*" execStart != null; +assert builtins.match ".*--health-listen 127.0.0.1:9332.*" execStart != null; +assert builtins.match ".*--metrics.*" execStart != null; +assert builtins.match ".*--health-listen.*" ( + defaultSystem.config.systemd.services.rbitcoin.serviceConfig.ExecStart +) == null; +assert builtins.match ".*--metrics.*" ( + defaultSystem.config.systemd.services.rbitcoin.serviceConfig.ExecStart +) == null; assert builtins.elem "tor.service" service.after; assert builtins.elem "tor.service" service.wants; assert builtins.elem "i2pd.service" service.after; @@ -208,7 +255,10 @@ assert builtins.match ".*--no-listen.*" listenOffExec != null; assert builtins.match ".*--rpc-socket.*" listenOffExec == null; assert builtins.match ".*--rpc-cookie-file.*" listenOffExec == null; assert !builtins.elem cookieTcpOnly (failedAssertions system); +assert !builtins.elem metricsNeedHealth (failedAssertions system); assert builtins.elem cookieTcpOnly (failedAssertions cookieWithoutTcp); +assert builtins.elem metricsNeedHealth (failedAssertions metricsWithoutHealth); +assert builtins.elem "127.0.0.1:9332" (builtins.head rbitcoinScrape.static_configs).targets; assert cookieDataDirMode == "0700"; assert builtins.match ".*--listen .*" listenOffExec == null; assert builtins.match ".*--max-inbound 0.*" listenOffExec != null; diff --git a/nix/tests/nixos-module-runtime.nix b/nix/tests/nixos-module-runtime.nix index 0e079b89e..c342e1b50 100644 --- a/nix/tests/nixos-module-runtime.nix +++ b/nix/tests/nixos-module-runtime.nix @@ -40,6 +40,8 @@ pkgs.testers.runNixOSTest { tor.control = "127.0.0.1:9051"; i2p.sam = "127.0.0.1:7656"; cjdns.reachable = true; + health.enable = true; + metrics = true; extraArgs = [ "--max-outbound" "4" @@ -84,6 +86,9 @@ pkgs.testers.runNixOSTest { machine.succeed("grep -Fx -- '--i2p-sam' /var/lib/rbitcoin-test/args") machine.succeed("grep -Fx -- '--cjdns-reachable' /var/lib/rbitcoin-test/args") machine.succeed("grep -Fx -- '127.0.0.1:7656' /var/lib/rbitcoin-test/args") + machine.succeed("grep -Fx -- '--health-listen' /var/lib/rbitcoin-test/args") + machine.succeed("grep -Fx -- '127.0.0.1:9332' /var/lib/rbitcoin-test/args") + machine.succeed("grep -Fx -- '--metrics' /var/lib/rbitcoin-test/args") machine.succeed("grep -Fx -- '--rpc-socket' /var/lib/rbitcoin-test/args") machine.succeed("grep -Fx -- '/run/rbitcoin/rpc.sock' /var/lib/rbitcoin-test/args") machine.succeed("test -e /run/rbitcoin/rpc.sock") From a6fb0883cd87ce040c18f9557ccd8a6298687a75 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:41:54 -0700 Subject: [PATCH 2/2] node: publish mempool min fee in sat/vB Operator surfaces use sat/vB. The scrape was copying the mempool's sat/kvB integer, which is the Core RPC unit. getmempoolinfo.mempoolminfee stays BTC/kvB. --- changelog.d/health-metrics-nixos.md | 5 +++-- crates/rbitcoin-node/src/health/metrics.rs | 6 +++--- crates/rbitcoin-test/tests/cross_surface.rs | 5 +++-- docs/operator/operations.md | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/changelog.d/health-metrics-nixos.md b/changelog.d/health-metrics-nixos.md index 0faf16125..606f14862 100644 --- a/changelog.d/health-metrics-nixos.md +++ b/changelog.d/health-metrics-nixos.md @@ -2,8 +2,9 @@ Added - **More Prometheus gauges.** `/metrics` also exposes verification progress, tip age, difficulty, peer counts by network, outbound time offset, P2P - byte totals, and mempool min fee, weight cap, orphans, and unbroadcast - count. Each one is a value RPC already publishes. + byte totals, and mempool min fee (sat/vB), weight cap, orphans, and + unbroadcast count. Fee rates on this scrape are sat/vB. The Core RPC + fields stay BTC/kvB. - **NixOS health and metrics.** `services.rbitcoin.health.enable` passes `--health-listen` (default `127.0.0.1:9332`). `services.rbitcoin.metrics` passes `--metrics` and adds a Prometheus scrape job when diff --git a/crates/rbitcoin-node/src/health/metrics.rs b/crates/rbitcoin-node/src/health/metrics.rs index 5b7d31214..b84d6394d 100644 --- a/crates/rbitcoin-node/src/health/metrics.rs +++ b/crates/rbitcoin-node/src/health/metrics.rs @@ -119,9 +119,9 @@ pub(super) fn render(status: &NodeStatus) -> String { vbytes, ); out.gauge( - "rbitcoin_mempool_min_fee_sat_per_kvb", - "Mempool min fee in sat/kvB (getmempoolinfo.mempoolminfee).", - mempool.mempool_min_fee_sat_kvb(), + "rbitcoin_mempool_min_fee_sat_per_vb", + "Minimum mempool feerate in sat/vB.", + mempool.mempool_min_fee_sat_kvb() as f64 / 1000.0, ); out.gauge( "rbitcoin_mempool_max_weight", diff --git a/crates/rbitcoin-test/tests/cross_surface.rs b/crates/rbitcoin-test/tests/cross_surface.rs index 1ff3c6d68..93998a85d 100644 --- a/crates/rbitcoin-test/tests/cross_surface.rs +++ b/crates/rbitcoin-test/tests/cross_surface.rs @@ -269,9 +269,10 @@ async fn pin_metrics_equal_rpc( ] { assert_eq!(m.get(series), Some(&want), "{series}: {m:?}"); } - let min_fee_sat = (num(&mempool["mempoolminfee"]) * 100_000_000.0).round(); + let min_fee_sat_kvb = (num(&mempool["mempoolminfee"]) * 100_000_000.0).round(); assert_eq!( - m["rbitcoin_mempool_min_fee_sat_per_kvb"], min_fee_sat, + m["rbitcoin_mempool_min_fee_sat_per_vb"], + min_fee_sat_kvb / 1000.0, "mempoolminfee: {mempool}" ); let now = std::time::SystemTime::now() diff --git a/docs/operator/operations.md b/docs/operator/operations.md index 805668964..cd6f8edd2 100644 --- a/docs/operator/operations.md +++ b/docs/operator/operations.md @@ -431,7 +431,7 @@ the min-fee, weight cap, orphan count, and unbroadcast count. NixOS: | `rbitcoin_network_receive_bytes_total` / `_transmit_bytes_total` | counter | `getnettotals.totalbytesrecv` / `totalbytessent` | | `rbitcoin_mempool_transactions` | gauge | `getmempoolinfo.size` | | `rbitcoin_mempool_bytes` | gauge | `getmempoolinfo.bytes` (virtual size) | -| `rbitcoin_mempool_min_fee_sat_per_kvb` | gauge | `getmempoolinfo.mempoolminfee` in sat/kvB | +| `rbitcoin_mempool_min_fee_sat_per_vb` | gauge | Minimum mempool feerate in sat/vB. `getmempoolinfo.mempoolminfee` is BTC/kvB | | `rbitcoin_mempool_max_weight` | gauge | `getmempoolinfo.maxmempool` (weight units) | | `rbitcoin_mempool_orphan_transactions` | gauge | `getmempoolinfo.orphanage.size` | | `rbitcoin_mempool_unbroadcast_transactions` | gauge | `getmempoolinfo.unbroadcastcount` |