diff --git a/internal/cli/export.go b/internal/cli/export.go index 8c00ef9..afaf541 100644 --- a/internal/cli/export.go +++ b/internal/cli/export.go @@ -29,6 +29,9 @@ operator address). Every alert carries the label profile=. --layout sidecar (default) a cometduty beside each node — everything, including disk, signer-state and log alerts that need the node's machine — plus a chain-role instance + --rpc adds endpoints the chain instance can reach (a + public RPC), so it can tell a chain halt from one + stuck node --layout standalone one cometduty for the whole network on the --chain-on host: every validator's signing, the active set, chain health, and each node whose RPC it can reach; no disk, @@ -118,12 +121,12 @@ cometduty doesn't need cometcli, and cometcli doesn't need cometduty.`, var sets []networktool.MonitorFiles switch layout { case "sidecar": - if rpcs != "" { - return fmt.Errorf("--rpc is for --layout standalone") - } - if sets, err = networktool.CometdutyFiles(chain, list, o); err != nil { + if sets, err = networktool.CometdutyFiles(chain, list, o, splitCSV(rpcs)); err != nil { return err } + if rpcs == "" { + fmt.Fprintln(w, "note: the chain instance sees one endpoint, so a stall there is reported on that node (node-stalled) — add --rpc for a chain-wide view") + } case "standalone": f, skipped, err := networktool.StandaloneFiles(chain, list, o, splitCSV(rpcs)) if err != nil { @@ -244,7 +247,7 @@ cometduty doesn't need cometcli, and cometcli doesn't need cometduty.`, f := cmd.Flags() f.StringVar(&format, "format", "cometduty", "what to write (cometduty)") f.StringVar(&layout, "layout", "sidecar", "sidecar (one per node + chain instance) or standalone (one for the network)") - f.StringVar(&rpcs, "rpc", "", "standalone: extra RPC endpoints it can reach, comma-separated (data sources, not paged)") + f.StringVar(&rpcs, "rpc", "", "extra RPC endpoints the chain (or standalone) instance can reach, comma-separated (data sources, not paged)") f.StringVar(&profiles, "profiles", "", "nodes to include (default: every profile on )") f.StringVar(&out, "out", "", "where to write (default ~/.cometcli/monitor/)") f.StringVar(&webhookURL, "webhook-url", "", "send alerts here, signed (e.g. where cometcli watch --listen is reachable)") diff --git a/internal/kb/cases/host.yaml b/internal/kb/cases/host.yaml index 84ef6ed..1b440a6 100644 --- a/internal/kb/cases/host.yaml +++ b/internal/kb/cases/host.yaml @@ -6,14 +6,16 @@ match: any: ["host.disk_used_pct >= 95", "logs.disk_io > 0"] confirm: - - "bash: df -h ; du -sh /data/* (largest dirs)" + - "bash: df -h ; du -xh -d 3 of the whole filesystem and files over 1G — the space can be outside the node home" causes: - unpruned state (app.pruning=nothing/default on a validator), old snapshots, huge logs, cosmovisor backups + - something else on the same filesystem (docker images and logs, backups, a stray large file) fix: - "[read] snap.prune advice; snap.list" - - "[change] remove old snapshots / rotated logs / cosmovisor backup dirs (operator confirms each delete)" + - "[change] remove old snapshots / rotated logs / cosmovisor backup dirs / the large files found (operator confirms each delete)" + - "[change] once space is free, restart the node (node.service restart): a node that hit a full disk freezes — RPC up, no new blocks, nothing logged — and stays frozen after space returns" - "[change] set app.toml pruning=custom (keep-recent 100, interval 10) for validators; restart" - - "[change] longer-term: grow the volume" + - "[change] longer-term: grow the volume, or move the node home to its own volume" verify: - "host.disk_used_pct < 85" warnings: @@ -21,7 +23,7 @@ - never delete application.db/blockstore.db on a validator to free space playbook_kind: report playbook: - - {do: bash, args: {command: "df -h / {sig:node.host_home}; du -sh {sig:node.host_home}/data/* 2>/dev/null | sort -h | tail -8", description: "what fills the disk"}, note: "where the space goes"} + - {do: bash, args: {command: "h={sig:node.host_home}; df -h $h; m=$(findmnt -no TARGET --target $h 2>/dev/null || echo /); if sudo -n true 2>/dev/null; then S='sudo -n'; else S=; fi; echo \"largest dirs on $m:\"; timeout 60 $S du -xh -d 3 $m 2>/dev/null | sort -h | tail -12; echo 'files over 1G:'; timeout 60 $S find $m -xdev -type f -size +1G -printf '%s\\t%p\\n' 2>/dev/null | sort -n | tail -8 | numfmt --to=iec --field=1", description: "what fills the disk (the whole filesystem, not just the node home)"}, note: "where the space goes — anything on the filesystem, not only the node data"} - {do: snap.prune, note: "pruning advice"} test: signals: {host.disk_used_pct: 98} @@ -36,7 +38,7 @@ - "[change] plan pruning / cleanup / resize before it fills" playbook_kind: report playbook: - - {do: bash, args: {command: "df -h / {sig:node.host_home}; du -sh {sig:node.host_home}/data/* 2>/dev/null | sort -h | tail -8", description: "what fills the disk"}, note: "where the space goes"} + - {do: bash, args: {command: "h={sig:node.host_home}; df -h $h; m=$(findmnt -no TARGET --target $h 2>/dev/null || echo /); if sudo -n true 2>/dev/null; then S='sudo -n'; else S=; fi; echo \"largest dirs on $m:\"; timeout 60 $S du -xh -d 3 $m 2>/dev/null | sort -h | tail -12; echo 'files over 1G:'; timeout 60 $S find $m -xdev -type f -size +1G -printf '%s\\t%p\\n' 2>/dev/null | sort -n | tail -8 | numfmt --to=iec --field=1", description: "what fills the disk (the whole filesystem, not just the node home)"}, note: "where the space goes — anything on the filesystem, not only the node data"} - {do: snap.prune, note: "pruning advice"} test: signals: {host.disk_used_pct: 88} @@ -55,6 +57,67 @@ test: signals: {host.inode_used_pct: 100} +- id: host-fs-readonly + title: Node filesystem went read-only + plain: "The disk holding the node switched to read-only, usually after disk errors. The node can't write and stops." + explains: [node-process-down, node-rpc-down, node-crash-loop, node-stalled, node-catching-up, node-restarted-recently, evm-rpc-down, val-not-signing, val-jail-imminent, val-signing-degraded, val-local-privval-errors, state-db-corrupt, host-disk-full] + severity: critical + match: + all: ["host.fs_readonly == true"] + confirm: + - "bash: findmnt --target ; journalctl -k | grep -iE 'I/O error|EXT4-fs error|read-only'" + causes: + - the kernel remounted the filesystem read-only after I/O or filesystem errors (errors=remount-ro) + - "a failing or detached volume (on a cloud VM: an impaired volume)" + fix: + - "[read] read the kernel log for the first error; check the volume's status at the provider" + - "[change] stop the validator so it can't restart mid-failure (node.service stop)" + - "[change] move the node to a healthy volume — copy the home, keep data/priv_validator_state.json at its latest — or fsck and remount read-write once the cause is fixed" + warnings: + - don't remount read-write without fsck; writes on a failing disk corrupt the databases + - never start a second copy of the validator elsewhere while this one may still sign + playbook_kind: report + playbook: + - {do: bash, args: {command: "h={sig:node.host_home}; findmnt --target $h; (sudo -n journalctl -k --since -6h -q --no-pager 2>/dev/null || journalctl -k --since -6h -q --no-pager 2>/dev/null) | grep -iE 'I/O error|EXT4-fs|XFS|BTRFS|read-only|nvme' | tail -15", description: "the mount and the kernel's disk errors"}, note: "why it went read-only"} + test: + signals: {host.fs_readonly: true} + +- id: host-disk-io-errors + title: Kernel reports disk I/O errors + plain: "The kernel logged disk errors in the last hour: the disk is failing or was detached." + explains: [node-process-down, node-rpc-down, node-crash-loop, node-stalled, node-blocksync-slow, node-restarted-recently, state-db-corrupt, val-signing-degraded, val-not-signing] + severity: critical + match: + all: ["host.kernel_disk_errors > 0"] + confirm: + - "bash: journalctl -k --since -1h | grep -iE 'I/O error|EXT4-fs error|nvme'" + causes: + - failing disk or controller; a cloud volume that is impaired or was detached + - filesystem corruption after an unclean shutdown + fix: + - "[read] which device and how often; the volume's status at the provider" + - "[change] move the node to a healthy volume before it fails outright (keep data/priv_validator_state.json at its latest)" + warnings: + - expect database corruption on a disk with I/O errors; restore from a snapshot rather than trusting the data + playbook_kind: report + playbook: + - {do: bash, args: {command: "(sudo -n journalctl -k --since -1h -q --no-pager 2>/dev/null || journalctl -k --since -1h -q --no-pager 2>/dev/null) | grep -iE 'I/O error|blk_update_request|medium error|EXT4-fs error|XFS|BTRFS|read-only|nvme' | tail -15; findmnt --target {sig:node.host_home}", description: "the kernel's disk errors"}, note: "which device is failing"} + test: + signals: {host.kernel_disk_errors: 7} + +- id: host-disk-failing + title: Disk health check failed + plain: "The disk's own health check (SMART) reports it is failing." + severity: critical + match: + all: ["host.disk_health_failed == true"] + confirm: + - "bash: sudo smartctl -a " + fix: + - "[change] replace the disk: move the node to a new one while this one still reads (keep data/priv_validator_state.json at its latest)" + test: + signals: {host.disk_health_failed: true} + - id: host-oom-killed title: Node killed for out-of-memory plain: "The machine ran out of memory and the kernel killed the node." diff --git a/internal/kb/cases/more.yaml b/internal/kb/cases/more.yaml index e0df31f..c668a66 100644 --- a/internal/kb/cases/more.yaml +++ b/internal/kb/cases/more.yaml @@ -96,7 +96,7 @@ - upgrades often run store migrations that need extra free space playbook_kind: report playbook: - - {do: bash, args: {command: "df -h / {sig:node.host_home}; du -sh {sig:node.host_home}/data/* 2>/dev/null | sort -h | tail -8", description: "what fills the disk"}, note: "where the space goes"} + - {do: bash, args: {command: "h={sig:node.host_home}; df -h $h; m=$(findmnt -no TARGET --target $h 2>/dev/null || echo /); if sudo -n true 2>/dev/null; then S='sudo -n'; else S=; fi; echo \"largest dirs on $m:\"; timeout 60 $S du -xh -d 3 $m 2>/dev/null | sort -h | tail -12; echo 'files over 1G:'; timeout 60 $S find $m -xdev -type f -size +1G -printf '%s\\t%p\\n' 2>/dev/null | sort -n | tail -8 | numfmt --to=iec --field=1", description: "what fills the disk (the whole filesystem, not just the node home)"}, note: "where the space goes — anything on the filesystem, not only the node data"} - {do: snap.prune, note: "pruning advice"} test: signals: {host.disk_free_gb: 8} diff --git a/internal/kb/cases/validator.yaml b/internal/kb/cases/validator.yaml index bdd00f1..eb0fa63 100644 --- a/internal/kb/cases/validator.yaml +++ b/internal/kb/cases/validator.yaml @@ -284,6 +284,30 @@ test: signals: {config.remote_signer: true, logs.privval: 40} +- id: val-signer-state-corrupt + title: priv_validator_state.json is corrupt + plain: "The validator's signing state file is corrupt. Starting it could double-sign, so cometcli won't start it." + explains: [node-process-down, node-crash-loop, val-not-signing, val-local-privval-errors] + severity: critical + match: + all: ["pvs.corrupt == true"] + confirm: + - "bash: cat /data/priv_validator_state.json" + causes: + - disk errors or a full disk while the file was written; a bad copy or restore + fix: + - "[read] find the last height this validator signed: its logs, or its last precommit on chain" + - "[change] write priv_validator_state.json with that height (or higher), round 0, step 0 — the operator writes it; never lower than what it signed" + - "[read] check the disk (host.kernel_disk_errors, host.fs_readonly) before trusting it again" + warnings: + - cometcli refuses to start, restart or recreate a validator on a corrupt state file + - never reset the file to height 0 on a validator that has signed + playbook_kind: report + playbook: + - {do: bash, args: {command: "f={sig:node.host_home}/data/priv_validator_state.json; ls -l $f; head -c 400 $f | od -c | head -12", description: "the state file as it is"}, note: "what's in the file"} + test: + signals: {pvs.corrupt: true} + - id: val-pvs-ahead title: priv_validator_state height ahead of the chain — signing refused plain: "The signing state is ahead of the chain; signing now could double-sign." diff --git a/internal/kb/kb_test.go b/internal/kb/kb_test.go index 574b338..7e2fe4b 100644 --- a/internal/kb/kb_test.go +++ b/internal/kb/kb_test.go @@ -51,7 +51,7 @@ func TestHealthyNodeMatchesNothing(t *testing.T) { "val.jailed": false, "val.tombstoned": false, "val.bonded": true, "val.status": "BONDED", "val.in_set": true, "val.signed_recent_pct": 100.0, "val.missed_pct": 0.1, "val.downtime_budget_used_pct": 0.2, "val.self_below_min": false, "val.fee_balance_zero": false, "upgrade.pending": false, "gov.unvoted": 0.0, - "host.disk_used_pct": 40.0, "host.inode_used_pct": 5.0, "host.mem_used_pct": 50.0, "host.load_per_cpu": 0.4, + "host.disk_used_pct": 40.0, "host.inode_used_pct": 5.0, "host.fs_readonly": false, "host.kernel_disk_errors": 0.0, "host.disk_health_failed": false, "pvs.corrupt": false, "host.mem_used_pct": 50.0, "host.load_per_cpu": 0.4, "clock.ntp_synced": true, "proc.running": true, "proc.restarts": 0.0, "proc.oom_killed": false, "proc.uptime_s": 86400.0, "logs.panic": 0.0, "logs.apphash": 0.0, "logs.peer_net": 12.0, "config.db_backend_mismatch": false, "config.mempool_mismatch": false, "config.remote_signer": false, diff --git a/internal/netspec/render.go b/internal/netspec/render.go index 9dda856..4edcd8e 100644 --- a/internal/netspec/render.go +++ b/internal/netspec/render.go @@ -282,6 +282,9 @@ func (c Compose) YAML() string { fmt.Fprintf(&b, " user: %q\n", c.User) } b.WriteString(" stop_grace_period: 60s\n working_dir: /data/node0/evmd\n environment:\n - ID=0\n - HOME=/data/node0/evmd\n") + // docker keeps a container's output forever by default: a node logging + // several lines a block fills GBs a day, and reading it back gets slow + b.WriteString(" logging:\n driver: json-file\n options: {max-size: \"100m\", max-file: \"5\"}\n") fmt.Fprintf(&b, " command: [%s]\n", strings.Join(cmd, ", ")) b.WriteString(" volumes:\n") fmt.Fprintf(&b, " - %s:/data/node0/evmd\n", c.HostHome) diff --git a/internal/netspec/render_test.go b/internal/netspec/render_test.go index d77342b..8e8d99e 100644 --- a/internal/netspec/render_test.go +++ b/internal/netspec/render_test.go @@ -73,7 +73,7 @@ func TestComposeFile(t *testing.T) { c := Compose{Image: "primium-v0.7.2", Container: "primium-archive", HostHome: "/data/archive", User: "1000:1000", Command: s.Command(Archive), Services: Services{API: true, GRPC: true, JSONRPC: true, WS: true}, PortOffset: 50} y := c.YAML() - for _, want := range []string{`"--pruning", "nothing"`, `"--json-rpc.ws-origins=*"`, `"26706:26656"`, `"127.0.0.1:26707:26657"`, `"127.0.0.1:8595:8545"`, "/data/archive:/data/node0/evmd", `user: "1000:1000"`, "stop_grace_period: 60s"} { + for _, want := range []string{`"--pruning", "nothing"`, `"--json-rpc.ws-origins=*"`, `"26706:26656"`, `"127.0.0.1:26707:26657"`, `"127.0.0.1:8595:8545"`, "/data/archive:/data/node0/evmd", `user: "1000:1000"`, "stop_grace_period: 60s", `max-size: "100m"`} { if !strings.Contains(y, want) { t.Errorf("compose lacks %s:\n%s", want, y) } diff --git a/internal/testutil/fakenode/fakedocker.go b/internal/testutil/fakenode/fakedocker.go index e354a02..e954689 100644 --- a/internal/testutil/fakenode/fakedocker.go +++ b/internal/testutil/fakenode/fakedocker.go @@ -9,9 +9,11 @@ import ( "os" "path/filepath" "regexp" + "slices" "strconv" "strings" "testing" + "time" "github.com/decred/dcrd/dcrec/secp256k1/v4" "github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa" @@ -85,7 +87,15 @@ func fakeDocker(args []string) int { case len(args) > 0 && args[0] == "logs": if p := os.Getenv("FAKENODE_LOGS"); p != "" { b, _ := os.ReadFile(p) - _, _ = os.Stdout.Write(b) + if !slices.Contains(args, "-t") { + _, _ = os.Stdout.Write(b) + return 0 + } + // -t: each line with its time, as docker prints it (all "now") + ts := time.Now().UTC().Format(time.RFC3339Nano) + for _, l := range strings.Split(strings.TrimSuffix(string(b), "\n"), "\n") { + fmt.Println(ts + " " + l) + } } return 0 case len(args) > 0 && args[0] == "inspect": diff --git a/internal/tools/common/signerstate.go b/internal/tools/common/signerstate.go new file mode 100644 index 0000000..8178ba8 --- /dev/null +++ b/internal/tools/common/signerstate.go @@ -0,0 +1,53 @@ +package common + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/abhijitkrm/cometcli/internal/client/host" + "github.com/abhijitkrm/cometcli/internal/toolkit" +) + +// SignerStateCorrupt reports whether a validator's +// data/priv_validator_state.json exists but doesn't parse. A missing file +// is not corrupt (a fresh node); a file that can't be read is not judged. +func SignerStateCorrupt(c *toolkit.Context, h host.Host) (bool, string) { + if c.Profile == nil || !c.Profile.IsValidator() { + return false, "" + } + raw, at, err := ReadNodeFile(c, h, "data/priv_validator_state.json") + if err != nil { + return false, "" + } + if why := ParseSignerState(raw); why != "" { + return true, fmt.Sprintf("%s is corrupt (%s)", at, why) + } + return false, "" +} + +// ParseSignerState returns why a priv_validator_state.json is unusable, or +// "" when it parses with a numeric height. +func ParseSignerState(raw []byte) string { + var pvs struct { + Height json.RawMessage `json:"height"` + } + if err := json.Unmarshal(raw, &pvs); err != nil { + return fmt.Sprintf("%d bytes, not JSON: %v", len(raw), err) + } + if _, err := strconv.ParseInt(strings.Trim(string(pvs.Height), `"`), 10, 64); err != nil { + return fmt.Sprintf("height %s isn't a number", OneLine(string(pvs.Height))) + } + return "" +} + +// GuardSignerState refuses to start a validator on a corrupt signer state: +// it can't know the last height it signed, so it may sign twice. +func GuardSignerState(c *toolkit.Context, h host.Host, what string) error { + if bad, why := SignerStateCorrupt(c, h); bad { + return fmt.Errorf("refusing to %s %s: %s — the validator can't tell what it last signed and could double-sign. "+ + "Find the last height it signed (its logs or the chain), write a state file at or above it, then retry", what, c.Profile.Name, why) + } + return nil +} diff --git a/internal/tools/common/signerstate_test.go b/internal/tools/common/signerstate_test.go new file mode 100644 index 0000000..2062fd9 --- /dev/null +++ b/internal/tools/common/signerstate_test.go @@ -0,0 +1,18 @@ +package common + +import "testing" + +func TestParseSignerState(t *testing.T) { + for raw, bad := range map[string]bool{ + `{"height":"12345","round":0,"step":3}`: false, + `{"height":678,"round":0,"step":3}`: false, + "{\"height\":\"1\x00\x00": true, + ``: true, + `{"height":"abc","round":0}`: true, + `{"round":0}`: true, + } { + if got := ParseSignerState([]byte(raw)) != ""; got != bad { + t.Errorf("ParseSignerState(%q) corrupt=%v, want %v", raw, got, bad) + } + } +} diff --git a/internal/tools/networktool/monitor.go b/internal/tools/networktool/monitor.go index e4fcdda..1dcf13b 100644 --- a/internal/tools/networktool/monitor.go +++ b/internal/tools/networktool/monitor.go @@ -20,8 +20,9 @@ import ( ) // CometdutyImage is the cometduty release the generated files run; it must -// understand labels, api_token and webhook.secret. -const CometdutyImage = "ghcr.io/abhijitkrm/cometduty:0.3.0" // goreleaser tags images without the v +// understand labels, api_token, webhook.secret, the inode and fill-rate +// alerts and the node host: block (0.4.0). +const CometdutyImage = "ghcr.io/abhijitkrm/cometduty:0.4.0" // goreleaser tags images without the v // MonitorOpts shape the cometduty files for a network. type MonitorOpts struct { @@ -89,7 +90,7 @@ func cometMetricsURL(rpc string) string { // CometdutyFiles renders a cometduty sidecar (config + compose) for each // profile, plus the chain-role instance beside opts.ChainOn. Profiles must // be on one chain. Nothing is written or run here. -func CometdutyFiles(chainID string, profiles []*config.Profile, o MonitorOpts) ([]MonitorFiles, error) { +func CometdutyFiles(chainID string, profiles []*config.Profile, o MonitorOpts, rpcs []string) ([]MonitorFiles, error) { if len(profiles) == 0 { return nil, fmt.Errorf("no nodes of %s", chainID) } @@ -129,6 +130,9 @@ func CometdutyFiles(chainID string, profiles []*config.Profile, o MonitorOpts) ( docker := p.Service.Type == "docker" && p.Service.Unit != "" if p.Home != "" { node["home"] = "/node-home" + // what a container can't see: the host's mount table and kernel + // log (DeployMonitor drops whichever the host doesn't have) + node["host"] = map[string]any{"mounts": hostMounts, "home_path": p.Home, "kernel_log": hostKernelLog} } if docker { node["logs"] = map[string]any{"enabled": true, "source": "docker", @@ -153,15 +157,21 @@ func CometdutyFiles(chainID string, profiles []*config.Profile, o MonitorOpts) ( "compose.yml": []byte(dutyCompose(o.Image, "cometduty-"+p.Name, p.Home, docker, 2375+i)), }}) } - // the chain-role instance: chain-wide alerts from one node's RPC + // the chain-role instance: chain-wide alerts from one node's RPC, plus + // any endpoints given (a public RPC) — with one endpoint a stall can't + // be told from that node being stuck, and cometduty says so port := o.MetricsPort + perHost[hostKey(chainNode)] + nodes := []any{map[string]any{"url": httpURL(chainNode.Endpoints.Comet), "name": chainNode.Name, + "labels": map[string]string{"profile": chainNode.Name}}} + for i, r := range rpcs { + nodes = append(nodes, map[string]any{"url": httpURL(r), "name": fmt.Sprintf("rpc-%d", i+1), "alert_if_down": false}) + } chain := map[string]any{ "chain_id": chainID, "role": "chain", "alerts": map[string]any{"stalled_enabled": true, "stalled_minutes": 2, "alert_if_no_servers": true, "set_watch_enabled": true, "webhook": map[string]any{"enabled": o.WebhookURL != ""}, "telegram": map[string]any{"enabled": o.TelegramChat != ""}}, - "nodes": []any{map[string]any{"url": httpURL(chainNode.Endpoints.Comet), "name": chainNode.Name, - "labels": map[string]string{"profile": chainNode.Name}}}, + "nodes": nodes, } cfgYAML, err := dutyConfig(o, port, map[string]any{chainID: chain}) if err != nil { @@ -199,9 +209,10 @@ func sidecarAlerts(o MonitorOpts) map[string]any { "lag_enabled": true, "lag_blocks": 100, "evm_down_enabled": true, "evm_lag_enabled": true, "evm_lag_blocks": 30, "catching_up_severity": "warning", "catching_up_grace_minutes": 2, "catching_up_stuck_minutes": 15, - "disk_free_bytes_alert": int64(5) << 30, "signer_stall_minutes": 5, - "webhook": map[string]any{"enabled": o.WebhookURL != ""}, - "telegram": map[string]any{"enabled": o.TelegramChat != ""}, + "disk_free_bytes_alert": int64(5) << 30, "inode_free_pct_alert": 5, "disk_fill_hours_alert": 12, + "signer_stall_minutes": 5, + "webhook": map[string]any{"enabled": o.WebhookURL != ""}, + "telegram": map[string]any{"enabled": o.TelegramChat != ""}, } } @@ -242,6 +253,10 @@ func dutyCompose(image, name, home string, logs bool, proxyPort int) string { b.WriteString(" volumes:\n - ./config.yml:/config/config.yml:ro\n - ./state:/data\n") if home != "" { fmt.Fprintf(&b, " - %s:/node-home:ro\n", home) + // the host's mount table and log dir (the dir, so a rotated + // kern.log is seen); the adm-style group that reads kern.log + fmt.Fprintf(&b, " - /proc/1/mounts:%s:ro\n - /var/log:%s:ro\n", hostMounts, path.Dir(hostKernelLog)) + b.WriteString(" group_add: [\"${LOG_GID:-0}\"]\n") } if logs { b.WriteString(" docker-proxy:\n image: tecnativa/docker-socket-proxy:0.3.0\n") @@ -277,6 +292,26 @@ func DeployMonitor(c *toolkit.Context, chainID string, f MonitorFiles, home stri owner = strings.TrimSpace(o) } } + // which host views this host has: a Linux mount table, a kern.log + // (and the group that reads it) + logGID := "0" + if f.Profile != "" { + caps, _, _ := h.Run(c, "test -r /proc/1/mounts && echo mounts; test -f /var/log/kern.log && echo kern=$(stat -c %g /var/log/kern.log 2>/dev/null)") + kern := "" + for _, l := range strings.Fields(caps) { + if v, ok := strings.CutPrefix(l, "kern="); ok { + kern = v + } + } + if kern != "" { + logGID = kern + } + files, err := fitHost(f.Files, strings.Contains(caps, "mounts"), kern != "") + if err != nil { + return "", err + } + f.Files = files + } if err := c.Approve(fmt.Sprintf("start cometduty on %s (%s, /metrics on :%d)", h, dir, f.Port), toolkit.TierLocalChange, map[string]any{"dir": dir, "image": CometdutyImage, "runs_as": owner, "metrics_port": f.Port}); err != nil { return "", err @@ -294,7 +329,7 @@ func DeployMonitor(c *toolkit.Context, chainID string, f MonitorFiles, home stri } // the container user must read config.yml _, _, _ = h.Run(c, fmt.Sprintf("chown %s %s/config.yml 2>/dev/null || sudo -n chown %s %s/config.yml", owner, common.ShellQ(dir), owner, common.ShellQ(dir))) - out, code, err := h.Run(c, fmt.Sprintf("cd %s && USER_ID=%s docker compose up -d 2>&1 | tail -3", common.ShellQ(dir), owner)) + out, code, err := h.Run(c, fmt.Sprintf("cd %s && USER_ID=%s LOG_GID=%s docker compose up -d 2>&1 | tail -3", common.ShellQ(dir), owner, logGID)) if err != nil || code != 0 { return "", fmt.Errorf("docker compose up failed (exit %d): %v %s", code, err, strings.TrimSpace(out)) } @@ -302,7 +337,7 @@ func DeployMonitor(c *toolkit.Context, chainID string, f MonitorFiles, home stri // cometduty must be restarted to read a new config (its state is kept) after, _, _ := h.Run(c, sum) if b := strings.TrimSpace(before); b != "" && b != strings.TrimSpace(after) { - if out, code, err := h.Run(c, fmt.Sprintf("cd %s && USER_ID=%s docker compose restart cometduty 2>&1 | tail -3", common.ShellQ(dir), owner)); err != nil || code != 0 { + if out, code, err := h.Run(c, fmt.Sprintf("cd %s && USER_ID=%s LOG_GID=%s docker compose restart cometduty 2>&1 | tail -3", common.ShellQ(dir), owner, logGID)); err != nil || code != 0 { return "", fmt.Errorf("restart cometduty for the new config (exit %d): %v %s", code, err, strings.TrimSpace(out)) } return fmt.Sprintf("cometduty restarted with the new config in %s (as %s), metrics on :%d", dir, owner, f.Port), nil @@ -376,8 +411,9 @@ func StandaloneFiles(chainID string, profiles []*config.Profile, o MonitorOpts, alerts := sidecarAlerts(o) alerts["stalled_enabled"], alerts["stalled_minutes"] = true, 2 alerts["alert_if_no_servers"], alerts["set_watch_enabled"] = true, true - delete(alerts, "disk_free_bytes_alert") // no node home here - delete(alerts, "signer_stall_minutes") + for _, k := range []string{"disk_free_bytes_alert", "inode_free_pct_alert", "disk_fill_hours_alert", "signer_stall_minutes"} { + delete(alerts, k) // no node home here + } chain := map[string]any{"chain_id": chainID, "alerts": alerts, "nodes": nodes} if len(validators) > 0 { chain["validators"] = validators @@ -418,3 +454,67 @@ func valoperOf(p *config.Profile) string { } return "" } + +// Where a sidecar sees its host (node host: in cometduty's config). +const ( + hostMounts = "/host/mounts" + hostKernelLog = "/host/log/kern.log" +) + +// fitHost drops the host views a host doesn't offer from a sidecar's files: +// no /proc/1/mounts (not Linux), or no readable /var/log/kern.log (a +// journald-only host — cometcli's triage reads the kernel journal there). +func fitHost(files map[string][]byte, mounts, kernLog bool) (map[string][]byte, error) { + out := map[string][]byte{} + for k, v := range files { + out[k] = v + } + if mounts && kernLog { + return out, nil + } + if raw, ok := out["config.yml"]; ok { + var root map[string]any + if err := yaml.Unmarshal(raw, &root); err != nil { + return nil, err + } + chains, _ := root["chains"].(map[string]any) + for _, ch := range chains { + nodes, _ := ch.(map[string]any)["nodes"].([]any) + for _, n := range nodes { + nm, _ := n.(map[string]any) + hc, _ := nm["host"].(map[string]any) + if hc == nil { + continue + } + if !mounts { + delete(hc, "mounts") + } + if !kernLog { + delete(hc, "kernel_log") + } + if hc["mounts"] == nil && hc["kernel_log"] == nil { + delete(nm, "host") + } + } + } + b, err := yaml.Marshal(root) + if err != nil { + return nil, err + } + head, _, _ := strings.Cut(string(raw), "\n") + out["config.yml"] = append([]byte(head+"\n"), b...) + } + if raw, ok := out["compose.yml"]; ok { + var keep []string + for _, l := range strings.Split(string(raw), "\n") { + switch { + case !mounts && strings.Contains(l, ":"+hostMounts+":"), + !kernLog && (strings.Contains(l, ":"+path.Dir(hostKernelLog)+":") || strings.Contains(l, "group_add:")): + continue + } + keep = append(keep, l) + } + out["compose.yml"] = []byte(strings.Join(keep, "\n")) + } + return out, nil +} diff --git a/internal/tools/networktool/monitor_test.go b/internal/tools/networktool/monitor_test.go index 2bfc14f..83979e9 100644 --- a/internal/tools/networktool/monitor_test.go +++ b/internal/tools/networktool/monitor_test.go @@ -28,7 +28,7 @@ func monitorProfiles() []*config.Profile { func TestCometdutyFiles(t *testing.T) { sets, err := CometdutyFiles("c-1", monitorProfiles(), MonitorOpts{WebhookURL: "https://cc.example/alerts", Secret: "sec", APIToken: "0123456789abcdef0123", ChainOn: "arch", - TelegramToken: "123:tok", TelegramChat: "-10042"}) + TelegramToken: "123:tok", TelegramChat: "-10042"}, nil) if err != nil { t.Fatal(err) } @@ -77,7 +77,7 @@ func TestCometdutyFiles(t *testing.T) { } func TestCometdutyFilesWithoutExtras(t *testing.T) { - sets, err := CometdutyFiles("c-1", monitorProfiles(), MonitorOpts{}) + sets, err := CometdutyFiles("c-1", monitorProfiles(), MonitorOpts{}, nil) if err != nil { t.Fatal(err) } @@ -88,7 +88,7 @@ func TestCometdutyFilesWithoutExtras(t *testing.T) { t.Errorf("%s: optional parts present without being asked:\n%s", s.Dir, c) } } - if _, err := CometdutyFiles("other-1", monitorProfiles(), MonitorOpts{}); err == nil { + if _, err := CometdutyFiles("other-1", monitorProfiles(), MonitorOpts{}, nil); err == nil { t.Error("profiles of another chain accepted") } } @@ -101,7 +101,7 @@ func TestCometdutyAcceptsExport(t *testing.T) { t.Skip("COMETDUTY_BIN not set") } sets, err := CometdutyFiles("c-1", monitorProfiles(), MonitorOpts{WebhookURL: "https://cc.example/alerts", Secret: "sec", APIToken: "0123456789abcdef0123", - TelegramToken: "123:tok", TelegramChat: "-10042"}) + TelegramToken: "123:tok", TelegramChat: "-10042"}, nil) if err != nil { t.Fatal(err) } @@ -155,7 +155,7 @@ func TestStandaloneFiles(t *testing.T) { } a := ch["alerts"].(map[string]any) if a["stalled_enabled"] != true || a["set_watch_enabled"] != true || a["alert_if_no_servers"] != true || - a["telegram"].(map[string]any)["enabled"] != true || a["disk_free_bytes_alert"] != nil { + a["telegram"].(map[string]any)["enabled"] != true || a["disk_free_bytes_alert"] != nil || a["disk_fill_hours_alert"] != nil { t.Errorf("alerts %v", a) } if c := string(f.Files["compose.yml"]); strings.Contains(c, "node-home") || strings.Contains(c, "docker-proxy") { @@ -170,3 +170,69 @@ func TestStandaloneFiles(t *testing.T) { t.Errorf("own host: %v", err) } } + +// A sidecar gets the host's mount table and kernel log; a host without +// them gets files without them (and nothing that names them). +func TestFitHost(t *testing.T) { + sets, err := CometdutyFiles("c-1", monitorProfiles(), MonitorOpts{}, nil) + if err != nil { + t.Fatal(err) + } + var side MonitorFiles + for _, s := range sets { + if s.Profile != "" && strings.Contains(string(s.Files["config.yml"]), "home_path") { + side = s + break + } + } + if side.Profile == "" { + t.Fatal("no sidecar with a node home") + } + cfg, comp := string(side.Files["config.yml"]), string(side.Files["compose.yml"]) + for _, want := range []string{"mounts: /host/mounts", "kernel_log: /host/log/kern.log"} { + if !strings.Contains(cfg, want) { + t.Errorf("config lacks %q:\n%s", want, cfg) + } + } + for _, want := range []string{"/proc/1/mounts:/host/mounts:ro", "/var/log:/host/log:ro", "group_add"} { + if !strings.Contains(comp, want) { + t.Errorf("compose lacks %q:\n%s", want, comp) + } + } + all, err := fitHost(side.Files, true, true) + if err != nil || string(all["config.yml"]) != cfg { + t.Fatalf("a host with both should keep the files as they are: %v", err) + } + none, err := fitHost(side.Files, false, false) + if err != nil { + t.Fatal(err) + } + c2, k2 := string(none["config.yml"]), string(none["compose.yml"]) + for _, gone := range []string{"/host/", "home_path", "group_add"} { + if strings.Contains(c2+k2, gone) { + t.Errorf("%q left on a host without them:\n%s\n%s", gone, c2, k2) + } + } + if !strings.HasPrefix(c2, "# cometduty config written by cometcli") || !strings.Contains(c2, "home: /node-home") { + t.Errorf("config lost its header or node home:\n%s", c2) + } + if string(side.Files["config.yml"]) != cfg { + t.Error("fitHost changed its input") + } + mountsOnly, _ := fitHost(side.Files, true, false) + if c := string(mountsOnly["config.yml"]); !strings.Contains(c, "mounts: /host/mounts") || strings.Contains(c, "kernel_log") { + t.Errorf("mounts only:\n%s", c) + } +} + +func TestChainInstanceTakesRPCs(t *testing.T) { + sets, err := CometdutyFiles("c-1", monitorProfiles(), MonitorOpts{}, []string{"https://rpc.example:443"}) + if err != nil { + t.Fatal(err) + } + for _, s := range sets { + if s.Profile == "" && !strings.Contains(string(s.Files["config.yml"]), "https://rpc.example:443") { + t.Fatalf("chain instance lacks the extra endpoint:\n%s", s.Files["config.yml"]) + } + } +} diff --git a/internal/tools/node/edit.go b/internal/tools/node/edit.go index 71dd360..78fa11e 100644 --- a/internal/tools/node/edit.go +++ b/internal/tools/node/edit.go @@ -134,6 +134,9 @@ func (recreateTool) Run(c *toolkit.Context, _ toolkit.Args) (*toolkit.Result, er default: return nil, fmt.Errorf("%s wasn't created by docker compose — no compose file to recreate it from", p.Service.Unit) } + if err := common.GuardSignerState(c, h, "recreate"); err != nil { + return nil, err + } if err := c.Approve(fmt.Sprintf("recreate %s (docker compose up -d --force-recreate %s, %s)", p.Service.Unit, svc, file), toolkit.TierLocalChange, map[string]any{"compose": file, "service": svc}); err != nil { return nil, err diff --git a/internal/tools/node/node.go b/internal/tools/node/node.go index 83064e3..c80725a 100644 --- a/internal/tools/node/node.go +++ b/internal/tools/node/node.go @@ -208,6 +208,11 @@ func (serviceTool) Run(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, err if err != nil { return nil, err } + if action == "start" || action == "restart" { + if err := common.GuardSignerState(c, h, action); err != nil { + return nil, err + } + } if action != "status" { if err := c.Approve(fmt.Sprintf("%s service %s on %s", action, c.Profile.Service.Unit, h), toolkit.TierLocalChange, map[string]any{"action": action, "unit": c.Profile.Service.Unit, "host": h.String()}); err != nil { diff --git a/internal/tools/triage/collect.go b/internal/tools/triage/collect.go index 3546cb3..0268f93 100644 --- a/internal/tools/triage/collect.go +++ b/internal/tools/triage/collect.go @@ -481,6 +481,7 @@ func collectHost(c *toolkit.Context, r *Report, set func(string, any)) error { } } } + collectHostDisk(home, run, set) if out := run("free -b 2>/dev/null | awk '/^Mem:/{print $2, $7}'"); out != "" { if fs := strings.Fields(out); len(fs) == 2 { total, _ := strconv.ParseFloat(fs[0], 64) @@ -749,7 +750,11 @@ func collectLogs(c *toolkit.Context, r *Report, set func(string, any)) error { var cmd string switch svc.Type { case "docker": - cmd = fmt.Sprintf("docker logs --since %s %s 2>&1 | tail -n 20000", since.String(), common.ShellQ(svc.Unit)) + // --since makes dockerd read a json-file log from its start (GBs on + // a node without log rotation, every sweep); --tail seeks from the + // end. Lines carry their time (-t), so the window is cut here. + cutoff := time.Now().Add(-since).UTC().Format("2006-01-02T15:04:05") + cmd = fmt.Sprintf("docker logs -t --tail 20000 %s 2>&1 | awk -v s=%s 'substr($0,1,19) >= s { print substr($0, index($0, \" \")+1) }'", common.ShellQ(svc.Unit), cutoff) case "systemd", "": unit := svc.Unit if unit == "" { @@ -894,6 +899,7 @@ func collectConfig(c *toolkit.Context, r *Report, set func(string, any)) error { } } if raw, err := readNodeFile(c, h, "data/priv_validator_state.json"); err == nil { + set("pvs.corrupt", common.ParseSignerState(raw) != "") var pvs struct { Height string `json:"height"` Round int64 `json:"round"` @@ -1236,3 +1242,39 @@ func dockerStartError(c *toolkit.Context, h host.Host, unit string, set func(str set("proc.image_present", false) } } + +// kernelDiskErr matches kernel log lines of a failing disk or filesystem. +const kernelDiskErr = `I/O error|blk_update_request: .*error|critical medium error|EXT4-fs error|XFS .*(corruption|Internal error|I/O error)|BTRFS (error|critical)|Remounting filesystem read-only|nvme[0-9]+: (I/O .*timeout|controller is down|Device not ready)` + +// collectHostDisk reads what a monitor inside a container can't see: whether +// the node home's filesystem went read-only, kernel disk errors in the last +// hour, and the disk's own health verdict (smartctl, when installed). +func collectHostDisk(home string, run func(string) string, set func(string, any)) { + q := common.ShellQ(home) + if opts := run("findmnt -no OPTIONS --target " + q + " 2>/dev/null"); opts != "" { + ro := false + for _, o := range strings.Split(opts, ",") { + ro = ro || o == "ro" + } + set("host.fs_readonly", ro) + } + // the kernel journal needs root or the adm group on most hosts: sudo + // (non-interactive), else the user's own view if it shows kernel lines + // at all — a quiet hour is 0 errors, a journal it can't read is unknown + // (dmesg has no time window) + if out := run(`if sudo -n true 2>/dev/null; then J="sudo -n journalctl"; elif [ -n "$(journalctl -k -n 1 -q --no-pager 2>/dev/null)" ]; then J=journalctl; else exit 1; fi; ` + + `$J -k --since "-1h" -q --no-pager 2>/dev/null | grep -cE '` + kernelDiskErr + `' || true`); out != "" { + if n, err := strconv.Atoi(out); err == nil { + set("host.kernel_disk_errors", float64(n)) + } + } + if dev := run("findmnt -no SOURCE --target " + q + " 2>/dev/null"); strings.HasPrefix(dev, "/dev/") { + out := run("command -v smartctl >/dev/null && sudo -n smartctl -H " + common.ShellQ(dev) + " 2>/dev/null | grep -iE 'overall-health|health status'") + switch u := strings.ToUpper(out); { + case strings.Contains(u, "FAILED"): + set("host.disk_health_failed", true) + case strings.Contains(u, "PASSED") || strings.Contains(u, ": OK"): + set("host.disk_health_failed", false) + } + } +}