Node/health endpoints

changelog.d/health-metrics-nixos.md

@@ -0,0 +1,11 @@
+Added
+
+- **More Prometheus gauges.** `/metrics` also exposes verification progress,
+  tip age, difficulty, peer counts by network, outbound time offset, P2P
+  byte totals, and mempool min fee (sat/vB), weight cap, orphans, and
+  unbroadcast count. Fee rates on this scrape are sat/vB. The Core RPC
+  fields stay BTC/kvB.
+- **NixOS health and metrics.** `services.rbitcoin.health.enable` passes
+  `--health-listen` (default `127.0.0.1:9332`). `services.rbitcoin.metrics`
+  passes `--metrics` and adds a Prometheus scrape job when
+  `services.prometheus.enable` is set.

crates/rbitcoin-consensus/src/header.rs

@@ -304,6 +304,22 @@ fn min_diff_bits(
     Some(bits)
 }
 
+/// Compact `nBits` as a difficulty relative to the max target (Core `GetDifficulty`).
+pub fn difficulty_from_bits(bits: u32) -> f64 {
+    let n_shift = ((bits >> 24) & 0xff) as i32;
+    let mut ddiff = (0x0000_ffff_u64 as f64) / ((bits & 0x00ff_ffff) as f64);
+    let mut shift = n_shift - 29;
+    while shift < 0 {
+        ddiff *= 256.0;
+        shift += 1;
+    }
+    while shift > 0 {
+        ddiff /= 256.0;
+        shift -= 1;
+    }
+    ddiff
+}
+
 fn header_bits_at(query: &Query, height: Height) -> Result<u32, ConsensusError> {
     if let Some((_fk, rec)) = query.header_at_height(height)? {
         return Ok(rec.bits);
@@ -324,6 +340,11 @@ mod median_time_past_tests {
     use rbitcoin_store::{HeaderRecord, InputRecord, OutputRecord, TxRecord};
     use std::time::{SystemTime, UNIX_EPOCH};
 
+    #[test]
+    fn max_target_bits_are_difficulty_one() {
+        assert_eq!(difficulty_from_bits(0x1d00_ffff), 1.0);
+    }
+
     #[test]
     fn pow_hash_meets_target_meet_miss_and_limit() {
         let rt = ChainParams::regtest();

crates/rbitcoin-consensus/src/lib.rs

@@ -70,7 +70,7 @@ pub use convert::header_to_record;
 pub(crate) use convert::{block_to_apply, block_to_apply_with_txids_prev};
 pub use error::{block_reject_log_line, block_reject_reason, script_flag_paren, ConsensusError};
 pub use header::{
-    expected_next_bits, median_time_past, next_work_bits, validate_header,
+    difficulty_from_bits, expected_next_bits, median_time_past, next_work_bits, validate_header,
     validate_header_on_parent,
 };
 pub use index_writebehind::{

crates/rbitcoin-net/src/lib.rs

@@ -60,9 +60,9 @@ pub use peer::{
 };
 pub use peer_dos::DEFAULT_MAX_INBOUND;
 pub use peers::{
-    connection_counts, parse_peer_addr, parse_peer_addr_with_port, parse_peer_net,
-    pick_stale_follow_evict, DialRequest, DialTarget, LivePeer, PeerConnType, PeerHub, PeerInfo,
-    PeerOut, PingAction,
+    connection_counts, outbound_time_offset, parse_peer_addr, parse_peer_addr_with_port,
+    parse_peer_net, pick_stale_follow_evict, DialRequest, DialTarget, LivePeer, PeerConnType,
+    PeerHub, PeerInfo, PeerOut, PingAction,
 };
 pub use perf_meter::RequestMeter;
 pub(crate) use rbitcoin_mempool::MempoolGraphStats;

crates/rbitcoin-net/src/peers.rs

@@ -1188,6 +1188,21 @@ pub fn connection_counts(peers: &[PeerInfo]) -> (u64, u64) {
     (inbound as u64, (peers.len() - inbound) as u64)
 }
 
+/// `getnetworkinfo.timeoffset`: median VERSION offset of completed outbound peers.
+/// Even counts use the upper middle. Inbound-only or empty is 0.
+pub fn outbound_time_offset(peers: &[PeerInfo]) -> i64 {
+    let mut offs: Vec<i64> = peers
+        .iter()
+        .filter(|p| !p.inbound && p.handshake_complete)
+        .map(|p| p.time_offset_secs)
+        .collect();
+    if offs.is_empty() {
+        return 0;
+    }
+    offs.sort_unstable();
+    offs[offs.len() / 2]
+}
+
 /// RPC-facing snapshot.
 #[derive(Clone, Debug)]
 pub struct PeerInfo {

crates/rbitcoin-node/src/health/metrics.rs

@@ -12,7 +12,8 @@ pub(super) const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"
 /// One scrape. Chain reads may touch the store and the mempool totals take
 /// its lock, so call from the blocking pool. Cost: those chain reads
 /// (`best_header_height`, tip header, `in_ibd`, scripthash lag), one peer
-/// snapshot, and one mempool fold (the same fold as `getmempoolinfo`).
+/// snapshot plus `byte_totals`, and one mempool fold (the same fold as
+/// `getmempoolinfo`) plus the min-fee, cap, orphan, and unbroadcast reads.
 pub(super) fn render(status: &NodeStatus) -> String {
     let mut out = Exposition::default();
     out.family(
@@ -67,14 +68,43 @@ pub(super) fn render(status: &NodeStatus) -> String {
         chain_gauges(&mut out, chain, status.sh_index);
     }
     if let Some(peers) = status.peers.get() {
-        let (inbound, outbound) = rbitcoin_net::connection_counts(&peers.snapshot());
+        let rows = peers.snapshot();
+        let (inbound, outbound) = rbitcoin_net::connection_counts(&rows);
         out.family(
             "rbitcoin_connections",
             "gauge",
             "Peer connections (getnetworkinfo.connections_in / connections_out).",
         );
         out.sample("rbitcoin_connections", "{direction=\"in\"}", inbound);
         out.sample("rbitcoin_connections", "{direction=\"out\"}", outbound);
+        out.gauge(
+            "rbitcoin_peer_time_offset_seconds",
+            "Median outbound peer clock offset (getnetworkinfo.timeoffset).",
+            rbitcoin_net::outbound_time_offset(&rows),
+        );
+        out.family(
+            "rbitcoin_peers",
+            "gauge",
+            "Peer connections by network (getpeerinfo.network).",
+        );
+        for network in ["ipv4", "ipv6", "onion", "i2p", "cjdns"] {
+            let n = rows
+                .iter()
+                .filter(|p| p.net.network_label() == network)
+                .count();
+            out.sample("rbitcoin_peers", &format!("{{network=\"{network}\"}}"), n);
+        }
+        let (recv, sent) = peers.byte_totals();
+        out.counter(
+            "rbitcoin_network_receive_bytes_total",
+            "P2P bytes received (getnettotals.totalbytesrecv).",
+            recv,
+        );
+        out.counter(
+            "rbitcoin_network_transmit_bytes_total",
+            "P2P bytes sent (getnettotals.totalbytessent).",
+            sent,
+        );
     }
     if let Some(mempool) = status.mempool.get() {
         let (size, vbytes, _fee) = mempool.live_adjusted_totals();
@@ -88,6 +118,27 @@ pub(super) fn render(status: &NodeStatus) -> String {
             "Sum of mempool virtual sizes (getmempoolinfo.bytes).",
             vbytes,
         );
+        out.gauge(
+            "rbitcoin_mempool_min_fee_sat_per_vb",
+            "Minimum mempool feerate in sat/vB.",
+            mempool.mempool_min_fee_sat_kvb() as f64 / 1000.0,
+        );
+        out.gauge(
+            "rbitcoin_mempool_max_weight",
+            "Mempool weight cap in weight units (getmempoolinfo.maxmempool).",
+            mempool.max_weight(),
+        );
+        let (orphans, _orphan_wu) = mempool.orphan_stats();
+        out.gauge(
+            "rbitcoin_mempool_orphan_transactions",
+            "Orphan transactions (getmempoolinfo.orphanage.size).",
+            orphans,
+        );
+        out.gauge(
+            "rbitcoin_mempool_unbroadcast_transactions",
+            "Local transactions not yet requested (getmempoolinfo.unbroadcastcount).",
+            mempool.unbroadcast_count(),
+        );
         let (accepts, rejects) = mempool.accept_totals();
         out.counter(
             "rbitcoin_mempool_accepts_total",
@@ -142,24 +193,54 @@ fn seconds(us: u64) -> f64 {
 
 fn chain_gauges(out: &mut Exposition, chain: &ChainHub, sh_index: bool) {
     let tip = chain.query.tip_height();
+    let blocks = tip.map_or(0, |h| h.0);
     out.gauge(
         "rbitcoin_blocks",
         "Active chain height (getblockchaininfo.blocks).",
-        tip.map_or(0, |h| h.0),
+        blocks,
     );
+    let headers = chain.best_header_height();
     out.gauge(
         "rbitcoin_headers",
         "Best header height (getblockchaininfo.headers).",
-        chain.best_header_height(),
+        headers,
     );
-    let time = tip
-        .and_then(|h| chain.query.header_at_height(h).ok().flatten())
-        .map_or(0, |(_, rec)| rec.timestamp);
+    let progress = if headers == 0 {
+        1.0
+    } else {
+        (f64::from(blocks) / f64::from(headers)).clamp(0.0, 1.0)
+    };
+    out.gauge(
+        "rbitcoin_verification_progress",
+        "blocks/headers (getblockchaininfo.verificationprogress).",
+        progress,
+    );
+    let rec = tip.and_then(|h| chain.query.header_at_height(h).ok().flatten());
+    let time = rec.as_ref().map_or(0, |(_, rec)| rec.timestamp);
     out.gauge(
         "rbitcoin_tip_time_seconds",
         "Tip block time (getblockchaininfo.time).",
         time,
     );
+    let age = rec.as_ref().map_or(0, |(_, rec)| {
+        chain
+            .clock
+            .now_secs()
+            .saturating_sub(u64::from(rec.timestamp))
+    });
+    out.gauge(
+        "rbitcoin_tip_age_seconds",
+        "Seconds since the tip block time.",
+        age,
+    );
+    let difficulty = rec.map_or(0.0, |(_, rec)| {
+        rbitcoin_consensus::difficulty_from_bits(rec.bits)
+    });
+    out.gauge(
+        "rbitcoin_difficulty",
+        "Tip difficulty (getblockchaininfo.difficulty).",
+        difficulty,
+    );
     out.gauge(
         "rbitcoin_initial_block_download",
         "1 during initial block download (getblockchaininfo.initialblockdownload).",

crates/rbitcoin-rpc/src/methods/chain.rs

@@ -133,21 +133,7 @@ pub(crate) fn rpc_warnings(ctx: &RpcContext) -> Vec<String> {
     w
 }
 
-pub(crate) fn difficulty_from_bits(bits: u32) -> f64 {
-    // Compact target → difficulty relative to max target (same class as Core).
-    let n_shift = ((bits >> 24) & 0xff) as i32;
-    let mut ddiff = (0x0000_ffff_u64 as f64) / ((bits & 0x00ff_ffff) as f64);
-    let mut shift = n_shift - 29;
-    while shift < 0 {
-        ddiff *= 256.0;
-        shift += 1;
-    }
-    while shift > 0 {
-        ddiff /= 256.0;
-        shift -= 1;
-    }
-    ddiff
-}
+pub(crate) use rbitcoin_consensus::difficulty_from_bits;
 
 pub(crate) fn difficulty_at_tip(ctx: &RpcContext) -> Result<f64, Value> {
     let tip = ctx

crates/rbitcoin-rpc/src/methods/net.rs

@@ -64,16 +64,7 @@ fn peer_block_connected(ctx: &RpcContext, hash: &bitcoin::BlockHash) -> bool {
 }
 
 fn outbound_median_time_offset(rows: &[rbitcoin_net::PeerInfo]) -> i64 {
-    let mut offs: Vec<i64> = rows
-        .iter()
-        .filter(|p| !p.inbound && p.handshake_complete)
-        .map(|p| p.time_offset_secs)
-        .collect();
-    if offs.is_empty() {
-        return 0;
-    }
-    offs.sort_unstable();
-    offs[offs.len() / 2]
+    rbitcoin_net::outbound_time_offset(rows)
 }
 
 pub(crate) fn peerinfo_json(ctx: &RpcContext, p: rbitcoin_net::PeerInfo) -> Value {

crates/rbitcoin-test/tests/cross_surface.rs

@@ -220,6 +220,8 @@ async fn pin_metrics_equal_rpc(
     let chain = jsonrpc(rpc_addr, "getblockchaininfo", json!([])).await["result"].clone();
     let net = jsonrpc(rpc_addr, "getnetworkinfo", json!([])).await["result"].clone();
     let mempool = jsonrpc(rpc_addr, "getmempoolinfo", json!([])).await["result"].clone();
+    let peers = jsonrpc(rpc_addr, "getpeerinfo", json!([])).await["result"].clone();
+    let totals = jsonrpc(rpc_addr, "getnettotals", json!([])).await["result"].clone();
     let m = scrape_metrics(health_addr).await;
     let num = |v: &Value| v.as_f64().unwrap_or_else(|| panic!("number: {v}"));
     let flag = |b: bool| if b { 1.0 } else { 0.0 };
@@ -249,14 +251,60 @@ async fn pin_metrics_equal_rpc(
         ),
         ("rbitcoin_mempool_transactions", num(&mempool["size"])),
         ("rbitcoin_mempool_bytes", num(&mempool["bytes"])),
+        (
+            "rbitcoin_verification_progress",
+            num(&chain["verificationprogress"]),
+        ),
+        ("rbitcoin_difficulty", num(&chain["difficulty"])),
+        ("rbitcoin_peer_time_offset_seconds", num(&net["timeoffset"])),
+        ("rbitcoin_mempool_max_weight", num(&mempool["maxmempool"])),
+        (
+            "rbitcoin_mempool_orphan_transactions",
+            num(&mempool["orphanage"]["size"]),
+        ),
+        (
+            "rbitcoin_mempool_unbroadcast_transactions",
+            num(&mempool["unbroadcastcount"]),
+        ),
     ] {
         assert_eq!(m.get(series), Some(&want), "{series}: {m:?}");
     }
-    assert!(m["rbitcoin_scripthash_lag_blocks"] <= 6.0, "{m:?}");
+    let min_fee_sat_kvb = (num(&mempool["mempoolminfee"]) * 100_000_000.0).round();
+    assert_eq!(
+        m["rbitcoin_mempool_min_fee_sat_per_vb"],
+        min_fee_sat_kvb / 1000.0,
+        "mempoolminfee: {mempool}"
+    );
     let now = std::time::SystemTime::now()
         .duration_since(std::time::UNIX_EPOCH)
         .unwrap()
         .as_secs_f64();
+    let tip_age = now - num(&chain["time"]);
+    assert!(
+        (m["rbitcoin_tip_age_seconds"] - tip_age).abs() < 30.0,
+        "tip age {} vs {tip_age}",
+        m["rbitcoin_tip_age_seconds"]
+    );
+    let peer_rows = peers.as_array().expect("getpeerinfo array");
+    for network in ["ipv4", "ipv6", "onion", "i2p", "cjdns"] {
+        let want = peer_rows.iter().filter(|p| p["network"] == network).count() as f64;
+        let series = format!("rbitcoin_peers{{network=\"{network}\"}}");
+        assert_eq!(m.get(&series), Some(&want), "{series}: {m:?}");
+    }
+    // Bytes can move between the RPC read and the scrape. Both are the same counters.
+    let recv = num(&totals["totalbytesrecv"]);
+    let sent = num(&totals["totalbytessent"]);
+    assert!(
+        (m["rbitcoin_network_receive_bytes_total"] - recv).abs() < 1_000_000.0,
+        "recv {} vs {recv}",
+        m["rbitcoin_network_receive_bytes_total"]
+    );
+    assert!(
+        (m["rbitcoin_network_transmit_bytes_total"] - sent).abs() < 1_000_000.0,
+        "sent {} vs {sent}",
+        m["rbitcoin_network_transmit_bytes_total"]
+    );
+    assert!(m["rbitcoin_scripthash_lag_blocks"] <= 6.0, "{m:?}");
     let started = m["process_start_time_seconds"];
     assert!(started <= now && started > now - 600.0, "{m:?}");
     #[cfg(any(target_os = "linux", target_os = "macos"))]

docs/operator/operations.md

@@ -407,8 +407,11 @@ which. Counters are the process-lifetime totals behind the 5s DEBUG
 `tip: perf` line; that line still prints only the change since its previous
 sample. A scrape runs on the blocking pool: chain reads
 (`best_header_height`, the tip header, `in_ibd`, and scripthash lag with
-`--sh-index`), one peer snapshot, and one mempool fold for
-`rbitcoin_mempool_bytes` (the same fold as `getmempoolinfo`).
+`--sh-index`), one peer snapshot and a byte-total walk, and one mempool
+fold for `rbitcoin_mempool_bytes` (the same fold as `getmempoolinfo`) plus
+the min-fee, weight cap, orphan count, and unbroadcast count. NixOS:
+`services.rbitcoin.health.enable` and `services.rbitcoin.metrics`
+([setup](setup.md#nixos-service)).
 
 | Metric | Type | Equals |
 |--------|------|--------|
@@ -419,9 +422,19 @@ sample. A scrape runs on the blocking pool: chain reads
 | `rbitcoin_headers` | gauge | `getblockchaininfo.headers` |
 | `rbitcoin_tip_time_seconds` | gauge | `getblockchaininfo.time` |
 | `rbitcoin_initial_block_download` | gauge | `getblockchaininfo.initialblockdownload` |
+| `rbitcoin_verification_progress` | gauge | `getblockchaininfo.verificationprogress` (`blocks/headers`) |
+| `rbitcoin_tip_age_seconds` | gauge | Seconds since `getblockchaininfo.time` |
+| `rbitcoin_difficulty` | gauge | `getblockchaininfo.difficulty` |
 | `rbitcoin_connections{direction="in"\|"out"}` | gauge | `getnetworkinfo.connections_in` / `connections_out` |
+| `rbitcoin_peers{network}` | gauge | Count of `getpeerinfo` rows for `ipv4`, `ipv6`, `onion`, `i2p`, `cjdns` |
+| `rbitcoin_peer_time_offset_seconds` | gauge | `getnetworkinfo.timeoffset` |
+| `rbitcoin_network_receive_bytes_total` / `_transmit_bytes_total` | counter | `getnettotals.totalbytesrecv` / `totalbytessent` |
 | `rbitcoin_mempool_transactions` | gauge | `getmempoolinfo.size` |
 | `rbitcoin_mempool_bytes` | gauge | `getmempoolinfo.bytes` (virtual size) |
+| `rbitcoin_mempool_min_fee_sat_per_vb` | gauge | Minimum mempool feerate in sat/vB. `getmempoolinfo.mempoolminfee` is BTC/kvB |
+| `rbitcoin_mempool_max_weight` | gauge | `getmempoolinfo.maxmempool` (weight units) |
+| `rbitcoin_mempool_orphan_transactions` | gauge | `getmempoolinfo.orphanage.size` |
+| `rbitcoin_mempool_unbroadcast_transactions` | gauge | `getmempoolinfo.unbroadcastcount` |
 | `rbitcoin_scripthash_lag_blocks` | gauge | `tip: accept sh_lag=` (with `--sh-index`) |
 | `rbitcoin_esplora_requests_total` / `_request_seconds_total` | counter | Lifetime sum of `tip: perf esplora req=`, and of that handler's wall time in seconds. The DEBUG line is the last ~5s window (`req=`, `avg_us` in microseconds) |
 | `rbitcoin_electrum_requests_total` / `_request_seconds_total` | counter | Lifetime sum of `tip: perf electrum req=`, and of that handler's wall time in seconds. The DEBUG line is the last ~5s window (`req=`, `avg_us` in microseconds) |

docs/operator/setup.md

@@ -89,6 +89,11 @@ enables the required scripthash index. `p2p.openFirewall`,
 JSON-RPC has no firewall option; expose it only through an explicitly managed
 firewall or tunnel.
 
+`health.enable` binds `--health-listen` (default `127.0.0.1:9332`) for
+`/healthz` and `/readyz`. `metrics` adds `--metrics` on that listener and,
+when `services.prometheus.enable` is set, a scrape job for the health
+address. Leave `health.openFirewall` off: the listener is unauthenticated.
+
 The daemon does not terminate TLS. Keep its application listeners on loopback
 and compose them with a proxy. This example serves Esplora and RPC over HTTPS,
 and Electrum as TLS-wrapped TCP on port 50002, using one ACME certificate: