Disk failures: host signals and cases, corrupt signer-state guard, whole-filesystem disk-full playbook, sidecar host mode

internal/cli/export.go

@@ -29,6 +29,9 @@ operator address). Every alert carries the label profile=<name>.
   --layout sidecar     (default) a cometduty beside each node — everything,
                        including disk, signer-state and log alerts that need
                        the node's machine — plus a chain-role instance
+                       --rpc adds endpoints the chain instance can reach (a
+                       public RPC), so it can tell a chain halt from one
+                       stuck node
   --layout standalone  one cometduty for the whole network on the --chain-on
                        host: every validator's signing, the active set, chain
                        health, and each node whose RPC it can reach; no disk,
@@ -118,12 +121,12 @@ cometduty doesn't need cometcli, and cometcli doesn't need cometduty.`,
 			var sets []networktool.MonitorFiles
 			switch layout {
 			case "sidecar":
-				if rpcs != "" {
-					return fmt.Errorf("--rpc is for --layout standalone")
-				}
-				if sets, err = networktool.CometdutyFiles(chain, list, o); err != nil {
+				if sets, err = networktool.CometdutyFiles(chain, list, o, splitCSV(rpcs)); err != nil {
 					return err
 				}
+				if rpcs == "" {
+					fmt.Fprintln(w, "note: the chain instance sees one endpoint, so a stall there is reported on that node (node-stalled) — add --rpc <a public RPC> for a chain-wide view")
+				}
 			case "standalone":
 				f, skipped, err := networktool.StandaloneFiles(chain, list, o, splitCSV(rpcs))
 				if err != nil {
@@ -244,7 +247,7 @@ cometduty doesn't need cometcli, and cometcli doesn't need cometduty.`,
 	f := cmd.Flags()
 	f.StringVar(&format, "format", "cometduty", "what to write (cometduty)")
 	f.StringVar(&layout, "layout", "sidecar", "sidecar (one per node + chain instance) or standalone (one for the network)")
-	f.StringVar(&rpcs, "rpc", "", "standalone: extra RPC endpoints it can reach, comma-separated (data sources, not paged)")
+	f.StringVar(&rpcs, "rpc", "", "extra RPC endpoints the chain (or standalone) instance can reach, comma-separated (data sources, not paged)")
 	f.StringVar(&profiles, "profiles", "", "nodes to include (default: every profile on <chain-id>)")
 	f.StringVar(&out, "out", "", "where to write (default ~/.cometcli/monitor/<chain-id>)")
 	f.StringVar(&webhookURL, "webhook-url", "", "send alerts here, signed (e.g. where cometcli watch --listen is reachable)")

internal/kb/cases/host.yaml

@@ -6,22 +6,24 @@
   match:
     any: ["host.disk_used_pct >= 95", "logs.disk_io > 0"]
   confirm:
-    - "bash: df -h <home>; du -sh <home>/data/* (largest dirs)"
+    - "bash: df -h <home>; du -xh -d 3 of the whole filesystem and files over 1G — the space can be outside the node home"
   causes:
     - unpruned state (app.pruning=nothing/default on a validator), old snapshots, huge logs, cosmovisor backups
+    - something else on the same filesystem (docker images and logs, backups, a stray large file)
   fix:
     - "[read] snap.prune advice; snap.list"
-    - "[change] remove old snapshots / rotated logs / cosmovisor backup dirs (operator confirms each delete)"
+    - "[change] remove old snapshots / rotated logs / cosmovisor backup dirs / the large files found (operator confirms each delete)"
+    - "[change] once space is free, restart the node (node.service restart): a node that hit a full disk freezes — RPC up, no new blocks, nothing logged — and stays frozen after space returns"
     - "[change] set app.toml pruning=custom (keep-recent 100, interval 10) for validators; restart"
-    - "[change] longer-term: grow the volume"
+    - "[change] longer-term: grow the volume, or move the node home to its own volume"
   verify:
     - "host.disk_used_pct < 85"
   warnings:
     - never delete data/priv_validator_state.json or anything under config/
     - never delete application.db/blockstore.db on a validator to free space
   playbook_kind: report
   playbook:
-    - {do: bash, args: {command: "df -h / {sig:node.host_home}; du -sh {sig:node.host_home}/data/* 2>/dev/null | sort -h | tail -8", description: "what fills the disk"}, note: "where the space goes"}
+    - {do: bash, args: {command: "h={sig:node.host_home}; df -h $h; m=$(findmnt -no TARGET --target $h 2>/dev/null || echo /); if sudo -n true 2>/dev/null; then S='sudo -n'; else S=; fi; echo \"largest dirs on $m:\"; timeout 60 $S du -xh -d 3 $m 2>/dev/null | sort -h | tail -12; echo 'files over 1G:'; timeout 60 $S find $m -xdev -type f -size +1G -printf '%s\\t%p\\n' 2>/dev/null | sort -n | tail -8 | numfmt --to=iec --field=1", description: "what fills the disk (the whole filesystem, not just the node home)"}, note: "where the space goes — anything on the filesystem, not only the node data"}
     - {do: snap.prune, note: "pruning advice"}
   test:
     signals: {host.disk_used_pct: 98}
@@ -36,7 +38,7 @@
     - "[change] plan pruning / cleanup / resize before it fills"
   playbook_kind: report
   playbook:
-    - {do: bash, args: {command: "df -h / {sig:node.host_home}; du -sh {sig:node.host_home}/data/* 2>/dev/null | sort -h | tail -8", description: "what fills the disk"}, note: "where the space goes"}
+    - {do: bash, args: {command: "h={sig:node.host_home}; df -h $h; m=$(findmnt -no TARGET --target $h 2>/dev/null || echo /); if sudo -n true 2>/dev/null; then S='sudo -n'; else S=; fi; echo \"largest dirs on $m:\"; timeout 60 $S du -xh -d 3 $m 2>/dev/null | sort -h | tail -12; echo 'files over 1G:'; timeout 60 $S find $m -xdev -type f -size +1G -printf '%s\\t%p\\n' 2>/dev/null | sort -n | tail -8 | numfmt --to=iec --field=1", description: "what fills the disk (the whole filesystem, not just the node home)"}, note: "where the space goes — anything on the filesystem, not only the node data"}
     - {do: snap.prune, note: "pruning advice"}
   test:
     signals: {host.disk_used_pct: 88}
@@ -55,6 +57,67 @@
   test:
     signals: {host.inode_used_pct: 100}
 
+- id: host-fs-readonly
+  title: Node filesystem went read-only
+  plain: "The disk holding the node switched to read-only, usually after disk errors. The node can't write and stops."
+  explains: [node-process-down, node-rpc-down, node-crash-loop, node-stalled, node-catching-up, node-restarted-recently, evm-rpc-down, val-not-signing, val-jail-imminent, val-signing-degraded, val-local-privval-errors, state-db-corrupt, host-disk-full]
+  severity: critical
+  match:
+    all: ["host.fs_readonly == true"]
+  confirm:
+    - "bash: findmnt --target <home>; journalctl -k | grep -iE 'I/O error|EXT4-fs error|read-only'"
+  causes:
+    - the kernel remounted the filesystem read-only after I/O or filesystem errors (errors=remount-ro)
+    - "a failing or detached volume (on a cloud VM: an impaired volume)"
+  fix:
+    - "[read] read the kernel log for the first error; check the volume's status at the provider"
+    - "[change] stop the validator so it can't restart mid-failure (node.service stop)"
+    - "[change] move the node to a healthy volume — copy the home, keep data/priv_validator_state.json at its latest — or fsck and remount read-write once the cause is fixed"
+  warnings:
+    - don't remount read-write without fsck; writes on a failing disk corrupt the databases
+    - never start a second copy of the validator elsewhere while this one may still sign
+  playbook_kind: report
+  playbook:
+    - {do: bash, args: {command: "h={sig:node.host_home}; findmnt --target $h; (sudo -n journalctl -k --since -6h -q --no-pager 2>/dev/null || journalctl -k --since -6h -q --no-pager 2>/dev/null) | grep -iE 'I/O error|EXT4-fs|XFS|BTRFS|read-only|nvme' | tail -15", description: "the mount and the kernel's disk errors"}, note: "why it went read-only"}
+  test:
+    signals: {host.fs_readonly: true}
+
+- id: host-disk-io-errors
+  title: Kernel reports disk I/O errors
+  plain: "The kernel logged disk errors in the last hour: the disk is failing or was detached."
+  explains: [node-process-down, node-rpc-down, node-crash-loop, node-stalled, node-blocksync-slow, node-restarted-recently, state-db-corrupt, val-signing-degraded, val-not-signing]
+  severity: critical
+  match:
+    all: ["host.kernel_disk_errors > 0"]
+  confirm:
+    - "bash: journalctl -k --since -1h | grep -iE 'I/O error|EXT4-fs error|nvme'"
+  causes:
+    - failing disk or controller; a cloud volume that is impaired or was detached
+    - filesystem corruption after an unclean shutdown
+  fix:
+    - "[read] which device and how often; the volume's status at the provider"
+    - "[change] move the node to a healthy volume before it fails outright (keep data/priv_validator_state.json at its latest)"
+  warnings:
+    - expect database corruption on a disk with I/O errors; restore from a snapshot rather than trusting the data
+  playbook_kind: report
+  playbook:
+    - {do: bash, args: {command: "(sudo -n journalctl -k --since -1h -q --no-pager 2>/dev/null || journalctl -k --since -1h -q --no-pager 2>/dev/null) | grep -iE 'I/O error|blk_update_request|medium error|EXT4-fs error|XFS|BTRFS|read-only|nvme' | tail -15; findmnt --target {sig:node.host_home}", description: "the kernel's disk errors"}, note: "which device is failing"}
+  test:
+    signals: {host.kernel_disk_errors: 7}
+
+- id: host-disk-failing
+  title: Disk health check failed
+  plain: "The disk's own health check (SMART) reports it is failing."
+  severity: critical
+  match:
+    all: ["host.disk_health_failed == true"]
+  confirm:
+    - "bash: sudo smartctl -a <device>"
+  fix:
+    - "[change] replace the disk: move the node to a new one while this one still reads (keep data/priv_validator_state.json at its latest)"
+  test:
+    signals: {host.disk_health_failed: true}
+
 - id: host-oom-killed
   title: Node killed for out-of-memory
   plain: "The machine ran out of memory and the kernel killed the node."

internal/kb/cases/more.yaml

@@ -96,7 +96,7 @@
     - upgrades often run store migrations that need extra free space
   playbook_kind: report
   playbook:
-    - {do: bash, args: {command: "df -h / {sig:node.host_home}; du -sh {sig:node.host_home}/data/* 2>/dev/null | sort -h | tail -8", description: "what fills the disk"}, note: "where the space goes"}
+    - {do: bash, args: {command: "h={sig:node.host_home}; df -h $h; m=$(findmnt -no TARGET --target $h 2>/dev/null || echo /); if sudo -n true 2>/dev/null; then S='sudo -n'; else S=; fi; echo \"largest dirs on $m:\"; timeout 60 $S du -xh -d 3 $m 2>/dev/null | sort -h | tail -12; echo 'files over 1G:'; timeout 60 $S find $m -xdev -type f -size +1G -printf '%s\\t%p\\n' 2>/dev/null | sort -n | tail -8 | numfmt --to=iec --field=1", description: "what fills the disk (the whole filesystem, not just the node home)"}, note: "where the space goes — anything on the filesystem, not only the node data"}
     - {do: snap.prune, note: "pruning advice"}
   test:
     signals: {host.disk_free_gb: 8}

internal/kb/cases/validator.yaml

@@ -284,6 +284,30 @@
   test:
     signals: {config.remote_signer: true, logs.privval: 40}
 
+- id: val-signer-state-corrupt
+  title: priv_validator_state.json is corrupt
+  plain: "The validator's signing state file is corrupt. Starting it could double-sign, so cometcli won't start it."
+  explains: [node-process-down, node-crash-loop, val-not-signing, val-local-privval-errors]
+  severity: critical
+  match:
+    all: ["pvs.corrupt == true"]
+  confirm:
+    - "bash: cat <home>/data/priv_validator_state.json"
+  causes:
+    - disk errors or a full disk while the file was written; a bad copy or restore
+  fix:
+    - "[read] find the last height this validator signed: its logs, or its last precommit on chain"
+    - "[change] write priv_validator_state.json with that height (or higher), round 0, step 0 — the operator writes it; never lower than what it signed"
+    - "[read] check the disk (host.kernel_disk_errors, host.fs_readonly) before trusting it again"
+  warnings:
+    - cometcli refuses to start, restart or recreate a validator on a corrupt state file
+    - never reset the file to height 0 on a validator that has signed
+  playbook_kind: report
+  playbook:
+    - {do: bash, args: {command: "f={sig:node.host_home}/data/priv_validator_state.json; ls -l $f; head -c 400 $f | od -c | head -12", description: "the state file as it is"}, note: "what's in the file"}
+  test:
+    signals: {pvs.corrupt: true}
+
 - id: val-pvs-ahead
   title: priv_validator_state height ahead of the chain — signing refused
   plain: "The signing state is ahead of the chain; signing now could double-sign."

internal/kb/kb_test.go

@@ -51,7 +51,7 @@ func TestHealthyNodeMatchesNothing(t *testing.T) {
 		"val.jailed": false, "val.tombstoned": false, "val.bonded": true, "val.status": "BONDED", "val.in_set": true,
 		"val.signed_recent_pct": 100.0, "val.missed_pct": 0.1, "val.downtime_budget_used_pct": 0.2, "val.self_below_min": false,
 		"val.fee_balance_zero": false, "upgrade.pending": false, "gov.unvoted": 0.0,
-		"host.disk_used_pct": 40.0, "host.inode_used_pct": 5.0, "host.mem_used_pct": 50.0, "host.load_per_cpu": 0.4,
+		"host.disk_used_pct": 40.0, "host.inode_used_pct": 5.0, "host.fs_readonly": false, "host.kernel_disk_errors": 0.0, "host.disk_health_failed": false, "pvs.corrupt": false, "host.mem_used_pct": 50.0, "host.load_per_cpu": 0.4,
 		"clock.ntp_synced": true, "proc.running": true, "proc.restarts": 0.0, "proc.oom_killed": false, "proc.uptime_s": 86400.0,
 		"logs.panic": 0.0, "logs.apphash": 0.0, "logs.peer_net": 12.0,
 		"config.db_backend_mismatch": false, "config.mempool_mismatch": false, "config.remote_signer": false,

internal/netspec/render.go

@@ -282,6 +282,9 @@ func (c Compose) YAML() string {
 		fmt.Fprintf(&b, "    user: %q\n", c.User)
 	}
 	b.WriteString("    stop_grace_period: 60s\n    working_dir: /data/node0/evmd\n    environment:\n      - ID=0\n      - HOME=/data/node0/evmd\n")
+	// docker keeps a container's output forever by default: a node logging
+	// several lines a block fills GBs a day, and reading it back gets slow
+	b.WriteString("    logging:\n      driver: json-file\n      options: {max-size: \"100m\", max-file: \"5\"}\n")
 	fmt.Fprintf(&b, "    command: [%s]\n", strings.Join(cmd, ", "))
 	b.WriteString("    volumes:\n")
 	fmt.Fprintf(&b, "      - %s:/data/node0/evmd\n", c.HostHome)

internal/netspec/render_test.go

@@ -73,7 +73,7 @@ func TestComposeFile(t *testing.T) {
 	c := Compose{Image: "primium-v0.7.2", Container: "primium-archive", HostHome: "/data/archive", User: "1000:1000",
 		Command: s.Command(Archive), Services: Services{API: true, GRPC: true, JSONRPC: true, WS: true}, PortOffset: 50}
 	y := c.YAML()
-	for _, want := range []string{`"--pruning", "nothing"`, `"--json-rpc.ws-origins=*"`, `"26706:26656"`, `"127.0.0.1:26707:26657"`, `"127.0.0.1:8595:8545"`, "/data/archive:/data/node0/evmd", `user: "1000:1000"`, "stop_grace_period: 60s"} {
+	for _, want := range []string{`"--pruning", "nothing"`, `"--json-rpc.ws-origins=*"`, `"26706:26656"`, `"127.0.0.1:26707:26657"`, `"127.0.0.1:8595:8545"`, "/data/archive:/data/node0/evmd", `user: "1000:1000"`, "stop_grace_period: 60s", `max-size: "100m"`} {
 		if !strings.Contains(y, want) {
 			t.Errorf("compose lacks %s:\n%s", want, y)
 		}

internal/testutil/fakenode/fakedocker.go

@@ -9,9 +9,11 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/decred/dcrd/dcrec/secp256k1/v4"
 	"github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa"
@@ -85,7 +87,15 @@ func fakeDocker(args []string) int {
 	case len(args) > 0 && args[0] == "logs":
 		if p := os.Getenv("FAKENODE_LOGS"); p != "" {
 			b, _ := os.ReadFile(p)
-			_, _ = os.Stdout.Write(b)
+			if !slices.Contains(args, "-t") {
+				_, _ = os.Stdout.Write(b)
+				return 0
+			}
+			// -t: each line with its time, as docker prints it (all "now")
+			ts := time.Now().UTC().Format(time.RFC3339Nano)
+			for _, l := range strings.Split(strings.TrimSuffix(string(b), "\n"), "\n") {
+				fmt.Println(ts + " " + l)
+			}
 		}
 		return 0
 	case len(args) > 0 && args[0] == "inspect":

internal/tools/common/signerstate.go

@@ -0,0 +1,53 @@
+package common
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/abhijitkrm/cometcli/internal/client/host"
+	"github.com/abhijitkrm/cometcli/internal/toolkit"
+)
+
+// SignerStateCorrupt reports whether a validator's
+// data/priv_validator_state.json exists but doesn't parse. A missing file
+// is not corrupt (a fresh node); a file that can't be read is not judged.
+func SignerStateCorrupt(c *toolkit.Context, h host.Host) (bool, string) {
+	if c.Profile == nil || !c.Profile.IsValidator() {
+		return false, ""
+	}
+	raw, at, err := ReadNodeFile(c, h, "data/priv_validator_state.json")
+	if err != nil {
+		return false, ""
+	}
+	if why := ParseSignerState(raw); why != "" {
+		return true, fmt.Sprintf("%s is corrupt (%s)", at, why)
+	}
+	return false, ""
+}
+
+// ParseSignerState returns why a priv_validator_state.json is unusable, or
+// "" when it parses with a numeric height.
+func ParseSignerState(raw []byte) string {
+	var pvs struct {
+		Height json.RawMessage `json:"height"`
+	}
+	if err := json.Unmarshal(raw, &pvs); err != nil {
+		return fmt.Sprintf("%d bytes, not JSON: %v", len(raw), err)
+	}
+	if _, err := strconv.ParseInt(strings.Trim(string(pvs.Height), `"`), 10, 64); err != nil {
+		return fmt.Sprintf("height %s isn't a number", OneLine(string(pvs.Height)))
+	}
+	return ""
+}
+
+// GuardSignerState refuses to start a validator on a corrupt signer state:
+// it can't know the last height it signed, so it may sign twice.
+func GuardSignerState(c *toolkit.Context, h host.Host, what string) error {
+	if bad, why := SignerStateCorrupt(c, h); bad {
+		return fmt.Errorf("refusing to %s %s: %s — the validator can't tell what it last signed and could double-sign. "+
+			"Find the last height it signed (its logs or the chain), write a state file at or above it, then retry", what, c.Profile.Name, why)
+	}
+	return nil
+}

internal/tools/common/signerstate_test.go

@@ -0,0 +1,18 @@
+package common
+
+import "testing"
+
+func TestParseSignerState(t *testing.T) {
+	for raw, bad := range map[string]bool{
+		`{"height":"12345","round":0,"step":3}`: false,
+		`{"height":678,"round":0,"step":3}`:     false,
+		"{\"height\":\"1\x00\x00":               true,
+		``:                                      true,
+		`{"height":"abc","round":0}`:            true,
+		`{"round":0}`:                           true,
+	} {
+		if got := ParseSignerState([]byte(raw)) != ""; got != bad {
+			t.Errorf("ParseSignerState(%q) corrupt=%v, want %v", raw, got, bad)
+		}
+	}
+}