diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d7296f1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ + +# Build artifacts and local files that should never enter the build context. +.git +.github +dist/ +netdiag +netdiag.exe +*.test +CLAUDE.md +docs/ +*.md +!go.mod diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f9cb0a2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,66 @@ +# syntax=docker/dockerfile:1 + +# ── Build ──────────────────────────────────────────────────────────────────── +FROM golang:1.24-alpine AS build + +WORKDIR /src + +# Dependencies first, so a source-only change does not re-download the module +# cache on every build. +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +ARG VERSION=docker +ARG COMMIT=unknown +ARG DATE=unknown + +# CGO off keeps the binary static, so it runs on a base image with no libc of +# the builder's vintage. -trimpath keeps build paths out of the binary. +RUN CGO_ENABLED=0 go build \ + -trimpath \ + -ldflags "-s -w -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" \ + -o /out/netdiag . + +# ── Capabilities ───────────────────────────────────────────────────────────── +# setcap has to run somewhere with libcap, and the final image should not carry +# a package manager just to install it. Do it in its own stage and copy the +# binary with its file capabilities intact. +FROM alpine:3.21 AS setcap + +RUN apk add --no-cache libcap +COPY --from=build /out/netdiag /out/netdiag + +# cap_net_raw is what ping, trace, discover and `scan --fast` need. Granting it +# to the binary means the container does not have to run as root, and the +# capability cannot leak to anything else in the image. +RUN setcap cap_net_raw+ep /out/netdiag + +# ── Runtime ────────────────────────────────────────────────────────────────── +# Alpine rather than distroless: file capabilities need a filesystem that +# preserves extended attributes through COPY, and having a shell in the image +# is worth more than the few MB for a tool people will want to exec into. +FROM alpine:3.21 + +# ca-certificates for the HTTPS commands (http, speedtest); the rest of netdiag +# speaks raw TCP, UDP and ICMP and needs nothing else. +RUN apk add --no-cache ca-certificates \ + && adduser -D -H -u 10001 netdiag + +COPY --from=setcap /out/netdiag /usr/local/bin/netdiag + +# Unprivileged. The binary carries exactly the one capability it needs, so +# there is no reason for the process to be root. +# +# Note for hardened deployments: `--cap-drop=ALL` alone will not start this +# image. Linux refuses to exec a file with permitted capabilities the process +# could never be granted, so dropping cap_net_raw produces an exec error rather +# than a netdiag that falls back to unprivileged scanning. Drop everything and +# add back the one capability instead: +# +# docker run --rm --cap-drop=ALL --cap-add=NET_RAW netdiag scan host --fast +USER netdiag + +ENTRYPOINT ["/usr/local/bin/netdiag"] +CMD ["--help"] diff --git a/ROADMAP.md b/ROADMAP.md index 94b9e24..513173f 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,138 +1,92 @@ # Roadmap -This document outlines the full engineering transformation plan for netdiag — from a solid one-shot CLI tool into a production-grade, portfolio-quality network diagnostics platform. +netdiag is a finished tool, not a work in progress. This document records what +was built, what was deliberately **cut**, and why. + +The original plan had seven phases, Phase 0 through Phase 6, and would have +turned a diagnostics CLI into a monitoring platform: a daemon, Prometheus metrics, a TUI dashboard, SQLite +persistence, and a gRPC agent mode. Three of those phases shipped. Four were +cut, on purpose, because a smaller finished tool is worth more than a large +unfinished one — and because most of what was planned already exists, done +better, in Prometheus and Grafana. + +| Phase | Status | +| ----- | ------ | +| Phase 0 — Foundation hardening | ✅ **Shipped** `v0.2.0` | +| Phase 1 — Monitor daemon, metrics, alerting | ❌ **Cut** | +| Phase 2 — TUI dashboard | ❌ **Cut** | +| Phase 3 — Raw socket SYN scanner | ✅ **Shipped** | +| Phase 4 — SQLite persistence and `analyze` | ❌ **Cut** | +| Phase 5 — gRPC agent mode | ❌ **Cut** | +| Phase 6 — Portfolio polish | ✅ **Shipped** | --- -## Current Version: 0.3.0 +## ✅ Phase 0 — Foundation hardening — SHIPPED `v0.2.0` -Core one-shot commands working: `ping`, `scan`, `trace`, `http`, `dig`, `whois`, `speedtest`, `discover`. +The architectural base: a `pkg/probe/` package with a `Prober` interface and a +universal `Result` type, so probes contain network logic and nothing else. -Phase 0 shipped in v0.2.0. v0.3.0 is a correctness and hardening release on top -of it. Phase 3 (the SYN scanner) has shipped on top of that and its numbers are -measured, not projected. Phases 1, 2, 4, 5 and 6 below are **not started**; -every command, flag, metric, and output sample in them describes intended future -work, not current behavior. +- **`pkg/probe/`** — all business logic moved out of `cmd/`, behind one + interface. +- **Typed results** — every probe returns a `Result` carrying an outcome and one + probe-specific payload, instead of printing. +- **JSON output** — `--json` wired up for every command, from the same value the + table renderer receives. +- **Structured logging** — `log/slog`, to stderr by default so stdout stays a + clean pipe. +- **Config file** — `~/.netdiag.yaml` via Viper, with `NETDIAG_` environment + overrides. +- **Tests** — port range parsing, ping severity, result marshaling. ---- - -## ✅ Phase 0 — Foundation Hardening `v0.2.0` — SHIPPED - -> **Goal:** Establish the architectural base everything else builds on. No new features visible to users — but every later phase depends on this. - -### What Changes - -- **`pkg/probe/` package** — Extract all business logic out of `cmd/` into a reusable package with a `Prober` interface and a universal `Result` type. This allows the monitor daemon, TUI, and gRPC agent to share probe logic without circular imports. -- **Typed Result system** — Every probe returns a `Result` struct (with `PingData`, `ScanData`, `HTTPData`, etc.) instead of printing directly. Enables JSON output, DB storage, and TUI rendering. -- **JSON output mode** — Wire up the existing `--json` flag (currently does nothing) so every command can output machine-readable JSON. -- **Structured logging** — Replace `color.Cyan(...)` calls with Go 1.21's `log/slog`. Adds log levels, JSON log format, and working `--log-file` support. -- **Config file support** — `~/.netdiag.yaml` for persistent defaults (interval, targets, thresholds, DB path, metrics port). -- **Test suite** — First real tests: `parsePortRange`, ping severity logic, Result marshaling. - -### Deliverable - -`go test ./...` passes. `netdiag ping google.com --json` outputs valid JSON. All existing commands work identically. +`v0.3.0` followed as a correctness and hardening release on the same base: exit +codes, signal handling, timeout bounds, and tests that bind to production +functions rather than reimplementations. --- -## 🟨 Phase 1 — Systems Engineer: Observability & Daemons `v0.4.0` +## ✅ Phase 3 — Raw socket SYN scanner — SHIPPED -> **Goal:** Prove you can build long-running, production-ready services. +> **Goal:** solve a hard technical problem with a measurable result. -### New Commands +### The problem -``` -netdiag monitor --target google.com --target 1.1.1.1 --interval 30s -netdiag monitor --config ~/.netdiag.yaml --alert-threshold 200ms -netdiag monitor --target google.com --webhook https://hooks.slack.com/... -``` - -### What's Built - -- **`pkg/monitor/` daemon** — Deterministic `time.Ticker` loop (not `time.Sleep`). Runs all probers concurrently via `errgroup` on every tick. -- **Graceful shutdown** — `signal.NotifyContext` handles `Ctrl+C` / `SIGTERM`, cancelling the entire goroutine tree cleanly. -- **Prometheus metrics server** — Embedded HTTP server on `:9090` exposing `netdiag_ping_latency_ms`, `netdiag_ping_packet_loss_percent`, `netdiag_http_status_code`, `netdiag_last_success_timestamp`, and more. -- **Alert subsystem** — `pkg/alert/` with `ConsoleAlerter`, `SlackAlerter`, `WebhookAlerter`. Cooldown mechanism prevents alert storms (per-host timestamp map with mutex). -- **JSONL event log** — Every probe result written to `~/.netdiag/logs/netdiag-YYYY-MM-DD.jsonl` (one JSON object per line, trivially parseable with `jq`). - -### Deliverable - -`netdiag monitor --target google.com` runs forever, logs results, exposes `http://localhost:9090/metrics`, and shuts down cleanly on `Ctrl+C`. - ---- - -## 🟦 Phase 2 — Frontend Engineer: TUI Dashboard `v0.5.0` - -> **Goal:** Build a "wow factor" interface that proves you understand complex, event-driven architecture. - -### New Command +`net.DialTimeout` completes a full three-way handshake per port — wasteful, and +it leaves completed connections in the target's logs. A SYN scan sends only the +initial SYN and reads the reply: SYN-ACK means open, RST means closed, silence +means filtered. The handshake is never completed. +```bash +netdiag scan 192.168.1.1 -p 1-65535 --fast # SYN scan +netdiag scan 127.0.0.1 -p 1-65535 --benchmark # compare both methods ``` -netdiag dashboard --target google.com --target 1.1.1.1 --target github.com -``` - -### Layout - -``` -┌─────────────────────────────────────────────────────────────────────┐ -│ netdiag dashboard [P]ause [Q]uit [↑↓] Select [Enter] Details │ -├────────────────────────────┬────────────────────────────────────────┤ -│ HOST TABLE │ LATENCY GRAPH (selected host) │ -│ ● google.com 12ms │ google.com — avg 12ms │ -│ ● 1.1.1.1 8ms │ 50ms ┤ │ -│ ⚠ github.com 145ms │ 25ms ┤ ▁▂▁▃▂▁▁▂▄▂▃▁▂▁▁▂▃▁▂▁ │ -│ ✗ badhost.io DOWN │ 0ms └──────────────────────── time │ -├────────────────────────────┴────────────────────────────────────────┤ -│ EVENT LOG [scroll ↑↓] │ -│ 10:00:01 ✓ google.com responded in 12ms │ -│ 10:00:31 ⚠ github.com latency spike: 145ms (threshold: 100ms) │ -│ 10:01:01 ✗ badhost.io — no response (timeout after 5s) │ -└─────────────────────────────────────────────────────────────────────┘ -``` - -### What's Built - -- **`charmbracelet/bubbletea`** — Elm-inspired model/update/view architecture. Background probe workers communicate via `tea.Cmd` — the correct pattern that avoids race conditions. -- **Sparkline graphs** — Unicode block characters (`▁▂▃▄▅▆▇█`) rendered from a 60-point ring buffer of latency history per host. -- **Split-pane layout** — `charmbracelet/lipgloss` for responsive terminal layout that handles resize events. -- **Detail view** — Press Enter on any host to see P50/P95/P99 stats, full latency graph, uptime %, and recent event history. -- **Keyboard navigation** — `↑↓` to select, `p` to pause, `q` to quit, `r` to force re-probe. - -### Deliverable - -`netdiag dashboard` opens a full-screen TUI with live-updating sparklines, scrollable event log, keyboard navigation, graceful resize, and clean exit on `q`. - ---- - -## ✅ Phase 3 — Low-Level Engineer: Raw Socket SYN Scanner `v0.6.0` — SHIPPED - -> **Goal:** Solve a hard technical problem with a measurable, benchmarkable result. - -### New Flag - -``` -netdiag scan 192.168.1.1 -p 1-65535 --fast # SYN scan -netdiag scan 192.168.1.1 -p 1-1024 --benchmark # compare both methods -``` - -### The Problem - -Current `net.DialTimeout("tcp", ...)` completes a full 3-way TCP handshake per port — wasteful, slow, and leaves connection logs on the target. A SYN scan sends only the initial SYN packet and reads the response (SYN-ACK = open, RST = closed) — never completing the handshake. ### What shipped -- **`pkg/probe/syn_scanner.go`** — raw TCP SYN packet crafting. `google/gopacket` lays out and decodes the header; the TCP checksum is computed over the IPv4 pseudo-header by netdiag's own code, because handing that to a library helper is the part worth being able to explain. Requires `cap_net_raw` or root. -- **Adaptive concurrency** — additive-increase/multiplicative-decrease over a fixed window, backing off only on windows that contain both replies and timeouts. Total silence means a filtered range, not congestion. -- **Benchmark mode** — `--benchmark` runs both methods against the same target and prints a comparison table. -- **Fallback** — a raw socket refused for lack of privilege falls back to the connect scan, with a notice on stderr. +- **`pkg/probe/syn_scanner.go`** — raw TCP SYN crafting. `gopacket` lays out and + decodes the header; the TCP checksum over the IPv4 pseudo-header is netdiag's + own code, because handing that to a library helper is the part worth being + able to explain. Needs `cap_net_raw`. +- **Correlation, not counting** — a raw socket receives every TCP segment on the + machine, including this process's own SYNs. A reply counts only if it arrives + on the scan's source port and acknowledges the sequence number sent to that + port. +- **Adaptive concurrency** — additive-increase/multiplicative-decrease over a + fixed window, backing off only on windows containing both replies and + timeouts. Total silence means a filtered range, not congestion. +- **Fallback** — no capability, or no route-derived source address, and the scan + degrades to the connect scanner with one notice on stderr. +- **Benchmark mode** — `--benchmark` runs both methods against the same target + and reports if the SYN half fell back, so an unprivileged benchmark cannot be + mistaken for a real comparison. ### Measured results -Full methodology, environment and caveats: [`docs/performance.md`](docs/performance.md). -Measured on bare metal — a 12th Gen Intel Core i7-12650H running Linux -7.0.0-28-generic, with `setcap cap_net_raw+ep` on the binary rather than root. -The loopback rows are the median of 5 runs; the filtered rows are the median -of 3. +Full methodology, environment and caveats: +[`docs/performance.md`](docs/performance.md). Measured on bare metal — a 12th Gen +Intel Core i7-12650H running Linux 7.0.0-28-generic, with `setcap cap_net_raw+ep` +on the binary rather than root. Loopback rows are the median of 5 runs; the +filtered rows the median of 3. | Target | Method | Time | Ports/sec | Speedup | | ------ | ------ | ---- | --------- | ------- | @@ -152,8 +106,7 @@ agreed on which ports were open in every loopback run. The first working version was *slower* than the connect scan (0.75x). The fix was not the packet library — building and checksumming all 65,535 packets costs 1.7 ms, half a percent of the scan — but sending from eight raw sockets instead -of one, since the kernel serializes writes per socket. `docs/performance.md` -has the component-by-component measurements. +of one, since the kernel serializes writes per socket. There is also a measured accuracy advantage under file descriptor pressure. Scanning 200 open ports with `ulimit -n 32` and `-c 500`, five runs: @@ -164,127 +117,105 @@ Scanning 200 open ports with `ulimit -n 32` and `-c 500`, five runs: | syn — open ports found | 200 | 200 | 200 | 200 | 200 | The connect scan needs a descriptor per port and reports an `EMFILE` failure as -a closed port, so it silently under-reports. The SYN scan uses one socket for -the whole scan. +a closed port, so it silently under-reports. The WAN case usually cited for SYN scanning — a remote host that drops packets to closed ports, making the connect scan pay a full timeout per port — **was not measured**, because this environment has no authorized remote target. It is untested, not proven. -### Deliverable - -`netdiag scan 127.0.0.1 -p 1-65535 --benchmark` prints the measured comparison table above. `docs/performance.md` documents methodology and caveats. - --- -## 🟥 Phase 4 — Data Engineer: Persistence & Analytics `v0.7.0` - -> **Goal:** Demonstrate data modeling, time-series queries, and statistical analysis. +## ✅ Phase 6 — Portfolio polish — SHIPPED -### New Command +- **[`docs/performance.md`](docs/performance.md)** — the SYN scanner benchmark: + problem, method, environment, real numbers, and the caveats that qualify them. +- **[`docs/architecture.md`](docs/architecture.md)** — Mermaid diagram of the + layering, plus the reasoning behind the `Prober`/`Result` design and the + exit-code contract. +- **`Dockerfile`** — multi-stage, ~28 MB, unprivileged user, `cap_net_raw` on + the binary so ICMP and SYN scanning work without running as root. +- **README** — rewritten around an Engineering Highlights section. +- **Demo recording** — `docs/demo.tape` for [vhs](https://github.com/charmbracelet/vhs). -``` -netdiag analyze # summary of all hosts -netdiag analyze --target google.com --window 24h # detailed report -netdiag analyze --worst 10 # worst performing hosts -netdiag analyze --target google.com --peak-hours # when is latency highest? -netdiag analyze --format csv > report.csv # export -``` +Deliberately **not** built, because they belong to the cut phases: Grafana +dashboards, `docker-compose.yml`, and a Prometheus scrape config. -### What's Built +--- -- **Embedded SQLite** — `modernc.org/sqlite` (pure Go, no CGO, cross-compiles cleanly). Schema stores every probe result with nanosecond timestamps. Indexed on `(target, timestamp DESC)` for fast time-range queries. -- **`pkg/store/` interface** — `SaveResult`, `GetHistory`, `GetStats`, `GetWorstHosts`, `GetPeakLatencyHours`, `Compact`. Fully mockable for tests. -- **Percentile queries** — P50/P95/P99 computed via SQLite `NTILE(100)` window functions (no external stats library needed). -- **Z-score anomaly detection** — `pkg/analyze/anomaly.go` flags probes where current latency is >2 standard deviations from the hourly baseline. Standard deviation computed in SQL: `SQRT(AVG(x²) - AVG(x)²)`. -- **Automatic retention** — Configurable `retention_days` in config. Background compaction job runs on monitor startup. +# Cut phases -### Sample Output +These were planned and are not being built. They are recorded here because a +decision not to build something is worth more to a reader than a list of +intentions. -``` -Network Health Report — Last 24 Hours -┌──────────────────┬────────┬────────┬───────┬───────┬──────────┐ -│ Host │ Uptime │ Probes │ Avg │ P95 │ Failures │ -├──────────────────┼────────┼────────┼───────┼───────┼──────────┤ -│ google.com │ 100% │ 2,880 │ 12ms │ 18ms │ 0 │ -│ github.com │ 99.97% │ 2,880 │ 45ms │ 120ms │ 1 │ -│ api.myapp.com │ 98.2% │ 2,880 │ 23ms │ 89ms │ 52 │ -└──────────────────┴────────┴────────┴───────┴───────┴──────────┘ -⚠ api.myapp.com has elevated failure rate (1.8%). Investigate. -``` +## ❌ Phase 1 — Monitor daemon, Prometheus metrics, alerting — CUT -### Deliverable +A `netdiag monitor` daemon with a ticker loop, an embedded Prometheus metrics +endpoint, Slack/webhook alerting with cooldowns, and a JSONL event log. -`netdiag analyze --window 7d` generates a health report with percentile stats, peak-hour analysis, and anomaly flags. All monitor results are automatically persisted. +**Why cut.** This is a worse version of software that already exists. Anyone who +wants netdiag's measurements scraped can wrap the existing `--json` output in +four lines of shell and a `node_exporter` textfile collector; anyone who wants +real alerting wants Alertmanager, not a webhook poster with a cooldown map. The +one genuinely novel piece — the probes — already exists and is already +scriptable. ---- +## ❌ Phase 2 — TUI dashboard — CUT -## 🔵 Phase 5 — Distributed Systems: Agent Mode `v0.8.0` +A full-screen `bubbletea` dashboard with sparklines, a split-pane layout, a +scrollable event log, and keyboard navigation. -> **Goal:** Multi-region latency monitoring via gRPC — the feature that separates "side project" from "distributed systems experience." +**Why cut.** The most fun thing on the list and the least useful. It would have +been the largest single body of code in the project, in service of watching +numbers that Grafana already draws better, and every future probe would have +owed it a rendering path. The `Result` type makes it straightforward for anyone +who wants it. -### New Commands +## ❌ Phase 4 — SQLite persistence and `analyze` — CUT -``` -# On a remote server (e.g. DigitalOcean droplet in Frankfurt) -netdiag agent --port 7777 --location "eu-west-1" --auth-token $SECRET - -# Locally, aggregate from multiple regions -netdiag monitor \ - --agent agent-us.example.com:7777 \ - --agent agent-eu.example.com:7777 \ - --agent agent-ap.example.com:7777 \ - --target google.com -``` +Embedded SQLite storage for every probe result, a `pkg/store/` interface, an +`analyze` command with percentiles, peak-hour analysis, and z-score anomaly +detection. -### What's Built +**Why cut.** It depends entirely on Phase 1: without a daemon writing results +continuously, there is nothing to analyze. Cutting the daemon cut the data +source, and a time-series schema with no time series is just a schema. -- **Protocol Buffers** — `proto/netdiag.proto` defines `RunProbe`, `StreamProbes`, `GetInfo` RPC methods. -- **gRPC agent server** — Listens for probe requests, executes them locally, streams results back with location metadata. -- **Aggregating monitor** — Fans out each probe to all connected agents, collects responses, correlates by target. -- **Multi-region TUI column** — Dashboard gets a third column showing per-region latency side by side: - ``` - google.com │ 🇺🇸 us-east 12ms ● │ 🇩🇪 eu-west 98ms ● │ 🇯🇵 ap 180ms ● - ``` +## ❌ Phase 5 — gRPC agent mode — CUT -### Deliverable +`netdiag agent` on remote hosts, protobuf definitions, and an aggregating +monitor fanning probes out to multiple regions. -Three `netdiag agent` instances running in different regions, with `netdiag dashboard` showing geographic latency breakdown in real time. +**Why cut.** The distributed-systems credential is real, but so is the cost: +protobuf toolchain, auth, TLS between agents, version skew between agent and +aggregator, and a deployment story — all to run probes that already run fine +over SSH. It also depended on Phase 1 for the aggregating side. --- -## ⬛ Phase 6 — Portfolio Polish `v1.0.0` - -> **Goal:** Make sure the engineering depth is visible before anyone reads the code. - -### What's Built +## Version history -- **`docs/performance.md`** — Full write-up of the SYN scanner: problem statement, methodology, benchmark environment, results table, and technical explanation. -- **`docs/architecture.md`** — Mermaid architecture diagram showing how CLI, monitor daemon, TUI, SQLite, Prometheus, and gRPC agent interact. -- **`deploy/prometheus.yml`** — Ready-to-use Prometheus scrape config. -- **`deploy/grafana-dashboard.json`** — Pre-built Grafana dashboard with latency time-series, packet loss heatmap, HTTP status history, and uptime gauges. -- **`Dockerfile`** — Multi-stage build. `setcap cap_net_raw+ep` so ICMP works without full root. -- **`deploy/docker-compose.yml`** — One `docker compose up` starts netdiag monitor + Prometheus + Grafana. -- **Demo GIF** — 30-second terminal recording (via `vhs`) at the top of the README showing the live dashboard. -- **README overhaul** — Leads with an "Engineering Highlights" table mapping each feature to the skill it demonstrates. +| Version | Phase | What landed | +| ------- | ----- | ----------- | +| `v0.1.0` | — | Initial release, all one-shot commands | +| `v0.2.0` | Phase 0 | `pkg/probe/` refactor, JSON output, config file, first tests | +| `v0.3.0` | — | Correctness and hardening: exit codes, signals, real tests | +| unreleased | Phases 3 + 6 | SYN scanner, `--fast`, `--benchmark`, measured docs, architecture doc, Docker | ---- - -## Version Summary +Phases 3 and 6 are merged and on `main`, but not yet tagged: the latest release +is `v0.3.0`, so `--fast` and `--benchmark` are available by building from source +or from the Docker image, not from a released binary. -| Version | Phase | Key Feature | -| -------- | ------- | -------------------------------------------------------- | -| `v0.1.0` | — | Initial release, all one-shot commands ✅ | -| `v0.2.0` | Phase 0 | `pkg/probe/` refactor, JSON output, config file, tests ✅ | -| `v0.3.0` | — | Correctness + hardening: exit codes, signals, real tests ✅ | -| `v0.4.0` | Phase 1 | `monitor` daemon, Prometheus metrics, alerting | -| `v0.5.0` | Phase 2 | `dashboard` TUI with sparklines | -| `v0.6.0` | Phase 3 | SYN scanner, `--fast` flag, benchmarks | -| `v0.7.0` | Phase 4 | SQLite persistence, `analyze` command, anomaly detection | -| `v0.8.0` | Phase 5 | gRPC agent mode, multi-region dashboard | -| `v1.0.0` | Phase 6 | Docker, Grafana, demo GIF, full documentation | +## What would actually be worth adding ---- +Small, self-contained, and in keeping with what netdiag already is: -**Last Updated:** 2026-08-01 +- A BPF filter on the SYN scanner's receive socket. It currently reads this + process's own outbound SYNs back off the raw socket — 21,061 of them in one + instrumented run — and filtering them in the kernel is the obvious next + optimization. Untried, so no claim about what it would save. +- MTR-style continuous latency measurement, as a one-shot command rather than a + daemon. +- Full IPv6 support across all commands (`dig AAAA` already works). +- IP geolocation, mDNS/Zeroconf discovery, PCAP export. diff --git a/Readme.md b/Readme.md index 66e5754..4b15a4a 100644 --- a/Readme.md +++ b/Readme.md @@ -1,479 +1,388 @@ -# netdiag 🌐 +# netdiag -Latest release -Go 1.24+ -License -CI status - +[![Latest release](https://img.shields.io/github/v/release/ARCoder181105/netdiag)](https://github.com/ARCoder181105/netdiag/releases) +[![Go 1.24+](https://img.shields.io/badge/Go-1.24+-00ADD8?style=flat&logo=go)](https://go.dev/) +[![CI](https://github.com/ARCoder181105/netdiag/actions/workflows/ci.yml/badge.svg)](https://github.com/ARCoder181105/netdiag/actions) +[![License](https://img.shields.io/github/license/ARCoder181105/netdiag)](LICENSE) -**netdiag** is a powerful, unified network diagnostic CLI tool built in Go. It combines the functionality of multiple network utilities (`ping`, `traceroute`, `nmap`, `dig`, `whois`, `speedtest`) into a single, fast, and easy-to-use command-line interface. - -## 🚀 Features - -- **🏓 Concurrent Ping** - Test connectivity to multiple hosts simultaneously -- **📡 Speed Test** - Measure your internet download/upload speeds -- **🗺️ Traceroute** - Discover the network path to any destination -- **🔍 Port Scanner** - Scan for open TCP ports with high-performance concurrency -- **🌐 HTTP Health Check** - Verify website status and SSL certificate validity -- **📋 DNS Lookup** - Query DNS records (A, AAAA, MX, TXT, NS, CNAME) -- **📖 WHOIS Lookup** - Retrieve domain registration information -- **🔎 Network Discovery** - Scan your local network for active devices - -## 📋 Table of Contents - -- [Installation](#-installation) -- [Uninstallation](#-uninstallation) -- [Quick Start](#-quick-start) -- [Global Flags](#-global-flags) -- [Exit Codes](#-exit-codes) -- [Responsible Use](#-responsible-use) -- [Commands Reference](#-commands-reference) -- [Configuration](#-configuration) -- [Architecture & Concepts](#-architecture--concepts) -- [Permissions](#-permissions) -- [Contributing](#-contributing) -- [License](#-license) - -## 🛠️ Installation - -### Quick Install (Recommended) - -**Linux/macOS:** +One network diagnostics CLI instead of eight. `ping`, `traceroute`, port +scanning, DNS, HTTP and TLS checks, WHOIS, speed test, and LAN discovery — with +consistent JSON output, consistent exit codes, and consistent behavior on +Ctrl+C across every one of them. ```bash -curl -fsSL https://raw.githubusercontent.com/ARCoder181105/netdiag/main/install.sh | bash -``` - -**Windows (PowerShell as Administrator):** - -```powershell -irm https://raw.githubusercontent.com/ARCoder181105/netdiag/main/install.ps1 | iex +netdiag ping google.com cloudflare.com +netdiag scan 192.168.1.1 -p 1-65535 --fast +netdiag http https://example.com --json | jq .http_data.tls_days_left ``` ---- + -
-📦 Package Managers -#### Go Install +## Contents -```bash -go install github.com/ARCoder181105/netdiag@latest -``` +- [Engineering highlights](#engineering-highlights) +- [Install](#install) +- [Commands](#commands) +- [Scripting: JSON and exit codes](#scripting-json-and-exit-codes) +- [Configuration](#configuration) +- [Permissions](#permissions) +- [Docker](#docker) +- [Architecture](#architecture) +- [Responsible use](#responsible-use) +- [Contributing](#contributing) -
+## Engineering highlights -
-⬇️ Download Pre-built Binaries +The parts worth reading the source for. -Download the latest release for your platform: +### Half-open SYN scanning, with the checksum done by hand -**[📥 Download Latest Release](https://github.com/ARCoder181105/netdiag/releases/latest)** +`scan --fast` sends a bare TCP SYN and reads the reply without ever completing +the handshake — SYN-ACK means open, RST means closed, silence means filtered. +The TCP checksum is computed over the IPv4 pseudo-header by netdiag's own code +rather than handed to a library helper, and it is tested against the RFC 1071 +worked example plus a vector generated independently in Python. -Available platforms: +Replies are correlated on source port *and* sequence number, not arrival order, +because a raw socket receives every TCP segment on the machine — including this +process's own outbound SYNs and the kernel's RSTs. -- Linux (amd64, arm64) -- macOS (Intel, Apple Silicon) -- Windows (amd64) +Measured on bare metal, 65,535 loopback ports, median of 5 runs: -After downloading: +| Concurrency | connect | syn | | +|---|---|---|---| +| `-c 100` | 264 ms | 151 ms | **1.75x** | +| `-c 2000` | 385 ms | 128 ms | **3.0x** | -**Linux/macOS:** +It is also more *accurate* under file descriptor pressure: the connect scan +needs one descriptor per port and reports an `EMFILE` failure as a closed port, +so at `ulimit -n 32` it found 200, 184, 166, 154 and 200 of 200 open ports +across five runs, where the SYN scan found all 200 every time. -```bash -chmod +x netdiag-* -sudo mv netdiag-* /usr/local/bin/netdiag +Full methodology, the caveats, and the two bugs that benchmarking found — +including the version that was *slower* than the connect scan, and why the fix +was not the packet library — are in [docs/performance.md](docs/performance.md). -# Linux only: Grant ICMP capabilities -sudo setcap cap_net_raw+ep /usr/local/bin/netdiag -``` +### ICMP privilege negotiation -**Windows:** +Raw ICMP needs `CAP_NET_RAW`. Most systems also offer unprivileged ICMP +datagram sockets — but Windows has none, and Linux gates them behind a sysctl +that some distributions leave empty. -- Extract `netdiag.exe` -- Move to `C:\Windows\System32\` or add to PATH +netdiag picks the mode most likely to work on the current platform, retries with +the other one on a permission error, and caches whichever worked for the rest of +the process, so a 1,024-host `discover` sweep pays that cost at most once. If +neither works, the error tells you the exact command to fix it instead of +printing `socket: permission denied`. -
+The same degrade-rather-than-fail pattern covers `scan --fast`: no capability, +or no route to derive a source address from, and it falls back to the connect +scanner with one notice on stderr. -
-🔨 Build from Source +### A severity contract that does not break pipelines -**Prerequisites:** +Every probe returns a typed `Result` with a `Severity` that describes *the +target*, separately from whether the probe itself could run. That split is what +makes the exit codes usable in scripts, and it is why a warning exits `0` — +see [Scripting](#scripting-json-and-exit-codes). -- Go 1.24 or higher -- Git +### Probes never print -```bash -# Clone repository -git clone https://github.com/ARCoder181105/netdiag.git -cd netdiag +All network logic lives behind one interface in `pkg/probe/`, returns a value, +and touches neither stdout nor `os.Exit`. `--json`, table rendering, structured +logging, and exit codes are each implemented once in the shared runner rather +than per command. Details in [docs/architecture.md](docs/architecture.md). -# Build -go build -o netdiag +## Install -# Install (optional) -sudo mv netdiag /usr/local/bin/ +**Linux / macOS** -# Linux: Grant ICMP capabilities -sudo setcap cap_net_raw+ep /usr/local/bin/netdiag +```bash +curl -fsSL https://raw.githubusercontent.com/ARCoder181105/netdiag/main/install.sh | bash ``` -
- ---- - -### Verify Installation +**Windows** (PowerShell as Administrator) -```bash -netdiag --version -netdiag --help +```powershell +irm https://raw.githubusercontent.com/ARCoder181105/netdiag/main/install.ps1 | iex ``` -### Quick Test +**Go** ```bash -# Test connectivity -netdiag ping google.com - -# Run speed test -netdiag speedtest - -# Scan ports -netdiag scan localhost -p 1-1000 +go install github.com/ARCoder181105/netdiag@latest ``` -## 🗑️ Uninstallation +Both install scripts are fetched from `main` and piped straight into a shell. +If you would rather not do that, read +[install.sh](install.sh) first, or use `go install` or a +[release binary](#pre-built-binaries) instead — releases are versioned and +publish checksums. + +
+Pre-built binaries, building from source, uninstalling -If you need to remove netdiag, you can use the provided uninstallation scripts or remove it manually. +### Pre-built binaries -### Quick Uninstall +[Download the latest release](https://github.com/ARCoder181105/netdiag/releases/latest) +for Linux (amd64/arm64), macOS (Intel/Apple Silicon), or Windows (amd64). -**Linux/macOS:** +Every release publishes a `checksums.txt`. Verify before installing — download +it alongside the binary, then: ```bash -curl -fsSL https://raw.githubusercontent.com/ARCoder181105/netdiag/main/uninstall.sh | bash +sha256sum --check --ignore-missing checksums.txt ``` -**Windows (PowerShell as Administrator):** - -```powershell -irm https://raw.githubusercontent.com/ARCoder181105/netdiag/main/uninstall.ps1 | iex +```bash +chmod +x netdiag-* +sudo mv netdiag-* /usr/local/bin/netdiag +sudo setcap cap_net_raw+ep /usr/local/bin/netdiag # Linux, for ICMP and --fast ``` ---- - -### Manual Uninstall +On Windows, extract `netdiag.exe` and put it somewhere on your `PATH`. -
-Removing netdiag from your system +### From source -#### If installed via Makefile or script: +Requires Go 1.24+. ```bash -# Using Makefile -make uninstall - -# Or manually remove the binary -sudo rm /usr/local/bin/netdiag # Linux/macOS +git clone https://github.com/ARCoder181105/netdiag.git +cd netdiag +make build # or: go build -o netdiag +sudo make install # installs to /usr/local/bin and applies setcap ``` -#### If installed via Go: +### Uninstall ```bash -rm $(go env GOPATH)/bin/netdiag +curl -fsSL https://raw.githubusercontent.com/ARCoder181105/netdiag/main/uninstall.sh | bash ``` -#### Windows: - -```powershell -# If installed to System32 -Remove-Item C:\Windows\System32\netdiag.exe - -# Or remove from your custom PATH location -``` +Or manually: `sudo make uninstall`, `sudo rm /usr/local/bin/netdiag`, or +`rm $(go env GOPATH)/bin/netdiag` if you installed with `go install`.
-## 🚀 Quick Start - -```bash -# Test connectivity to multiple hosts -netdiag ping google.com cloudflare.com - -# Run an internet speed test -netdiag speedtest - -# Trace the route to a destination -netdiag trace github.com +Verify with `netdiag --version`. -# Scan for open ports -netdiag scan 192.168.1.1 --ports 1-1024 +## Commands -# Check website health and SSL certificate -netdiag http https://example.com +| Command | What it does | Needs privileges | +|---|---|---| +| [`ping`](#ping) | ICMP echo to one or more hosts, concurrently | usually not — see [Permissions](#permissions) | +| [`scan`](#scan) | TCP port scan, connect or half-open SYN | only for `--fast` | +| [`trace`](#trace) | Traceroute to a destination | yes | +| [`http`](#http) | HTTP status, latency, and TLS certificate check | no | +| [`dig`](#dig) | DNS lookups (A, AAAA, MX, TXT, NS, CNAME) | no | +| [`whois`](#whois) | Domain registration lookup | no | +| [`discover`](#discover) | Sweep the local network for active devices | usually not — see [Permissions](#permissions) | +| [`speedtest`](#speedtest) | Download and upload throughput | no | -# Lookup DNS records -netdiag dig google.com MX +`ping` and `discover` try unprivileged ICMP datagram sockets first and work +without setup on macOS and most Linux systems. `trace` and `scan --fast` always +need `CAP_NET_RAW` or the equivalent. -# Get domain registration info -netdiag whois example.com +Global flags, valid on every command: -# Discover devices on your local network -netdiag discover +```text + -j, --json Output machine-readable JSON instead of tables + -l, --log-file string Append structured logs to a file instead of stderr + --log-format string Log format: text or json (default "text") + --log-level string Log level: debug, info, warn, error (default "info") ``` -### 💻 Advanced JSON Parsing - -`netdiag` natively supports JSON output for all commands using the `--json` flag. To parse and filter this output in the terminal, we highly recommend installing [jq](https://jqlang.github.io/jq/). +`--version` is on `netdiag` itself, not on subcommands. -**Example:** Get the average ping latency: +### ping -```bash -netdiag ping 1.1.1.1 --json | jq '.[0].ping_data.avg_rtt' -``` - -## 🌍 Global Flags - -These work on every command: +Sends ICMP echo requests to any number of hosts concurrently. ```text - -j, --json Output machine-readable JSON instead of tables - -l, --log-file string Append structured logs to a file instead of stderr - --log-format string Log format: text or json (default: "text") - --log-level string Log level: debug, info, warn, error (default: "info") + -c, --count int Number of ICMP packets to send (default 3) + -t, --timeout duration Total timeout for the whole run, not per packet (default 5s) + -i, --interval duration Time to wait between packets (default 1s) + --concurrency int Number of hosts to ping concurrently (default 20) ``` -`-v, --version` is available on `netdiag` itself (`netdiag --version`), not on -subcommands. - -Logs always go to stderr (or `--log-file`), never stdout, so `--json` output -stays pipeable: - ```bash -netdiag ping 1.1.1.1 --json --log-level debug | jq '.[0].ping_data.avg_rtt' +netdiag ping google.com +netdiag ping -c 10 8.8.8.8 1.1.1.1 +netdiag ping -t 2s -i 500ms github.com ``` -## 🔢 Exit Codes +Reports packet loss and min/avg/max/stddev latency per host. Exits non-zero if +any host failed. -| Code | Severity | Meaning | -| ---- | ------------------ | ----------------------------------------------------------- | -| `0` | `OK` or `Warning` | The probe ran; the target is up | -| `1` | `Error` | The probe ran and the target failed | -| `2` | — | Invalid arguments, flags, or configuration | -| `3` | — | The probe could not run (no privileges, unresolvable host) | +### scan -**Warnings exit `0` on purpose.** A certificate expiring in 10 days, 25% packet -loss on a host that is still up, or a traceroute that did not reach the final -hop are all degraded-but-alive states. They are reported in the output and in -`severity`, but they do not fail the command — otherwise every warning would -break a pipeline: +Scans TCP ports, either with ordinary connections or with half-open SYN probes. -```bash -netdiag http https://api.example.com && ./deploy.sh +```text + -p, --ports string Ports to scan: a list, a range, or both (default "1-1024") + -t, --timeout duration Connection timeout per port (default 1s) + -c, --concurrency int Number of ports to probe concurrently (default 100) + --fast Use a half-open SYN scan (needs CAP_NET_RAW; falls back) + --benchmark Run both scan methods against the target and compare them ``` -To treat warnings as failures, check `severity` yourself (`0` OK, `1` Warning, -`2` Error, `3` Unknown): - ```bash -sev=$(netdiag http https://api.example.com --json | jq .severity) -[ "$sev" -eq 0 ] || exit 1 +netdiag scan localhost +netdiag scan 192.168.1.1 -p 80,443,8000-9000 +netdiag scan 192.168.1.1 -p 1-65535 --fast +netdiag scan 127.0.0.1 -p 1-65535 --benchmark ``` -## ⚠ Responsible Use +`--fast` needs `CAP_NET_RAW`; without it the scan falls back to the connect +method, says so once on stderr, and still returns results. `--benchmark` runs +both methods against the same target and prints a comparison — and tells you if +the SYN half fell back, so an unprivileged benchmark cannot be mistaken for a +real comparison. -`netdiag scan` and `netdiag discover` send unsolicited traffic to hosts. Scanning -or sweeping systems you do not own, or do not have explicit written permission to -test, is unlawful in many jurisdictions. Use these commands on your own -infrastructure or with documented authorization. - -## 📖 Commands Reference +A scan that finds nothing is a warning, not an error: it ran fine, there was +just nothing to report. A scan interrupted with Ctrl+C says its results are +incomplete rather than claiming the unscanned ports were closed. -### `netdiag ping` +### trace -Send ICMP echo requests to one or more hosts concurrently. +```text + -m, --max-hops int Maximum number of hops (default 30) + -t, --timeout duration Timeout per hop (default 2s) +``` ```bash -netdiag ping [more hosts...] - -Flags: - -c, --count int Number of ICMP packets to send (default: 3) - -t, --timeout duration Timeout per host, e.g. 1s, 500ms (default: 1s) - -i, --interval duration Time to wait between packets, e.g. 1s (default: 1s) - --concurrency int Number of hosts to ping concurrently (default: 20) - -Examples: - netdiag ping google.com - netdiag ping -c 10 8.8.8.8 1.1.1.1 - netdiag ping -t 2s -i 500ms github.com +netdiag trace google.com +netdiag trace 8.8.8.8 -m 20 ``` -**Output**: Displays a table with packet loss, average/min/max latency for each host. +Prints each hop with its IP, resolved hostname, and round-trip time. Replies are +matched by parsing the quoted original header out of the ICMP body and checking +the echo ID and sequence, so concurrent pings elsewhere on the machine cannot +pollute the hop list. ---- +### http -### `netdiag speedtest` - -Test your internet connection speed (download/upload). +```text + -t, --timeout duration Timeout for the request (default 5s) + -m, --method string HTTP method for the request (default "GET") + --skip-tls Skip TLS certificate verification (insecure) +``` ```bash -netdiag speedtest - -Flags: - -u, --no-upload Skip upload test - -s, --server string Specify server ID - -Examples: - netdiag speedtest - netdiag speedtest --no-upload - netdiag speedtest --server 12345 +netdiag http example.com +netdiag http https://github.com +netdiag http https://expired.badssl.com --timeout 10s ``` -**Output**: Shows ISP info, server details, ping, download speed, and upload speed with quality assessment. +Reports status code, latency, redirect count, and certificate issuer plus days +remaining. A certificate close to expiry is a warning, so it is visible without +failing a deploy pipeline. ---- +### dig -### `netdiag trace` - -Perform a traceroute to discover the network path to a destination. +```text + -s, --server string Custom DNS server (e.g. 8.8.8.8 or 8.8.8.8:5353) + -t, --timeout duration Query timeout (default 5s) +``` ```bash -netdiag trace - -Flags: - -m, --max-hops int Maximum number of hops (default: 30) - -t, --timeout duration Timeout per hop, e.g. 2s, 500ms (default: 2s) - -Examples: - netdiag trace google.com - netdiag trace 8.8.8.8 -m 20 +netdiag dig google.com # A records by default +netdiag dig google.com AAAA +netdiag dig github.com MX +netdiag dig google.com --server 1.1.1.1 ``` -**Output**: Displays each hop with IP address, hostname, and round-trip time. - ---- +Supported types: `A`, `AAAA`, `MX`, `TXT`, `NS`, `CNAME`. -### `netdiag scan` +### whois -Scan a target host for open TCP ports using a high-performance worker pool. +```text + -t, --timeout duration Query timeout (default 10s) +``` ```bash -netdiag scan - -Flags: - -p, --ports string Ports to scan: list, range, or both (default: "1-1024") - -t, --timeout duration Connection timeout per port (default: 1s) - -c, --concurrency int Ports to probe concurrently (default: 100) - -Examples: - netdiag scan localhost - netdiag scan 192.168.1.1 -p 80,443,8000-9000 - netdiag scan example.com -p 1-65535 +netdiag whois example.com ``` -**Output**: Lists all discovered open ports in a table format. - ---- +The timeout bounds the whole IANA → registry → registrar chain, not each hop +individually. -### `netdiag http` +### discover -Check HTTP status and SSL certificate information for a website. +```text + -t, --timeout duration Ping timeout per host (default 500ms) +``` ```bash -netdiag http - -Flags: - -t, --timeout duration Timeout for the request, e.g. 5s, 500ms (default: 5s) - -m, --method string HTTP method (default: "GET") - --skip-tls Skip TLS certificate verification (insecure) - -Examples: - netdiag http example.com - netdiag http https://github.com - netdiag http https://expired.badssl.com --timeout 10s +netdiag discover +netdiag discover -t 1s ``` -**Output**: - -- HTTP status code (color-coded by result) -- Request latency -- SSL certificate details (subject, issuer, validity period, expiration warning) +Detects your primary IPv4 network by asking the kernel which source address it +would use for outbound traffic — so a Docker bridge does not get swept instead +of your actual LAN — then sweeps it, capped at 1,024 addresses. ---- +### speedtest -### `netdiag dig` - -Perform DNS lookups for various record types. +```text + -u, --no-upload Skip upload test + -s, --server string Specify speedtest server ID +``` ```bash -netdiag dig [type] - -Supported Types: A, AAAA, MX, TXT, NS, CNAME - -Flags: - -s, --server string Custom DNS server (e.g. 8.8.8.8 or 8.8.8.8:5353) - -t, --timeout duration Query timeout (default: 5s) - -Examples: - netdiag dig google.com # Default: A records (IPv4) - netdiag dig google.com AAAA # IPv6 addresses - netdiag dig github.com MX # Mail servers - netdiag dig example.com TXT # Text records - netdiag dig google.com NS # Name servers - netdiag dig google.com --server 1.1.1.1 +netdiag speedtest +netdiag speedtest --no-upload ``` -**Output**: Table of DNS records matching the specified type. - ---- - -### `netdiag whois` +## Scripting: JSON and exit codes -Retrieve domain registration and ownership information. +Every command supports `--json`. Logs and diagnostics go to stderr by default — +or to a file with `--log-file` — and never to stdout, so the JSON stays a clean +pipe either way. The examples below use [jq](https://jqlang.github.io/jq/), +which is not required to run netdiag but makes the JSON output far easier to +work with: ```bash -netdiag whois - -Flags: - -t, --timeout duration Query timeout (default: 10s) - -Examples: - netdiag whois google.com - netdiag whois github.com +netdiag ping 1.1.1.1 --json | jq '.[0].ping_data.avg_rtt' +netdiag scan localhost -p 1-1024 --json | jq '.scan_data.open_ports' +netdiag http https://example.com --json | jq '.http_data.tls_days_left' ``` -**Output**: Full WHOIS record including registrar, creation date, expiration date, and nameservers. +| Code | Severity | Meaning | +|---|---|---| +| `0` | `OK` or `Warning` | The probe ran; the target is up | +| `1` | `Error` | The probe ran and the target failed | +| `2` | — | Invalid arguments, flags, or configuration | +| `3` | — | The probe could not run at all (for example, ICMP is not permitted) | ---- +A host that cannot be resolved is a failed *target*, not a probe that could not +run, so it exits `1` rather than `3`. -### `netdiag discover` - -Scan your local network for active devices using ping sweeps. +**Warnings exit `0` deliberately.** A certificate with 10 days left, 25% packet +loss on a host that still answers, or a traceroute that never reached the final +hop are degraded-but-alive states. They appear in the output and in `severity`, +but they do not fail the command — otherwise every warning would break this: ```bash -netdiag discover - -Flags: - -t, --timeout duration Ping timeout per host (default: 500ms) - -Examples: - netdiag discover - netdiag discover -t 1s +netdiag http https://api.example.com && ./deploy.sh ``` -**Output**: - -- Auto-detects your local IPv4 network, including its netmask -- Sweeps every usable host address in that network (capped at 1024 addresses) -- Displays table of discovered devices with IP, hostname, and latency +To treat warnings as failures, read `severity` yourself (`0` OK, `1` Warning, +`2` Error, `3` Unknown): ---- +```bash +sev=$(netdiag http https://api.example.com --json | jq .severity) +[ "$sev" -eq 0 ] || exit 1 +``` -## ⚙ Configuration +## Configuration -netdiag reads `~/.netdiag.yaml` if present. CLI flags always override it. +netdiag reads `~/.netdiag.yaml` if it exists. CLI flags always win. ```yaml scan: @@ -481,217 +390,126 @@ scan: default_timeout: "1s" ``` -Every key can also be set via the environment with a `NETDIAG_` prefix: +Any key can be set through the environment with a `NETDIAG_` prefix: ```bash NETDIAG_SCAN_DEFAULT_TIMEOUT=2s netdiag scan localhost ``` -Keys are added to the config schema only once a command actually reads them — -see [config.example.yaml](config.example.yaml) for the current set. - -## 🏗️ Architecture & Concepts +Keys enter the schema only once a command actually reads them — see +[config.example.yaml](config.example.yaml) for the current set. -### Layering +## Permissions -netdiag separates *what to measure* from *how to display it*. Commands are thin -Cobra wrappers; all network logic lives in `pkg/probe/`. - -```text -main.go - └── cmd/ Cobra commands: flags, argument validation, rendering - ├── root.go global flags, logger + config wiring - └── run.go runProbe(): the shared execution path - │ - ▼ - pkg/probe/ all network logic; no printing, no os.Exit - ├── types.go Result, Severity, the Prober interface - ├── ping.go PingProber - ├── scan.go ConnectScanner - ├── tracer.go TraceProber - ├── http.go HTTPProber - ├── dig.go DigProber - ├── discover.go DiscoverProber - ├── whois.go WhoisProber - ├── speedtest.go SpeedTestProber - └── icmp.go shared privileged/unprivileged ICMP handling - │ - ▼ - pkg/output/ color, tables, JSON - pkg/logger/ log/slog wrapper (stderr by default) - pkg/config/ Viper loader for ~/.netdiag.yaml -``` +Only `ping`, `trace`, `discover`, and `scan --fast` need anything special. +`scan` (default), `http`, `dig`, `whois`, and `speedtest` never do. -### The `Prober` interface +**Linux.** `ping` and `discover` try unprivileged ICMP datagram sockets first, +so they often work with no setup. If your kernel does not allow them, or you +want `trace` and `scan --fast`: -Every probe implements the same two methods: +```bash +# Preferred: grant only the capability this binary needs +sudo setcap cap_net_raw+ep /usr/local/bin/netdiag -```go -type Prober interface { - Probe(ctx context.Context) (Result, error) - Type() string -} +# Or: allow unprivileged ICMP for all users (does not help --fast or trace) +sudo sysctl -w net.ipv4.ping_group_range="0 2147483647" ``` -This is what lets one shared runner drive most commands. `ping` targets multiple -hosts, so it runs a batch loop over the same helpers (`logResult`, -`exitForAll`) instead of calling `runProbe` directly. +`make install` applies the `setcap` step. netdiag prints whichever of these +applies when it hits a permission error. -### The `Result` type +**macOS.** Unprivileged ICMP works out of the box for `ping` and `discover`. +`trace` and `scan --fast` need `sudo`. -Probes never print. They return a typed `Result` carrying an outcome plus one -probe-specific payload: +**Windows.** Run the terminal as Administrator for the ICMP commands. -```go -type Result struct { - TimeStamp time.Time - ProbeType string - Target string +## Docker - PingData *PingData // only one payload is non-nil - ScanData *ScanData - HTTPData *HTTPData - // ... DNSData, TraceData, DiscoverData, SpeedTestData, WhoisData - - Message string - Severity Severity // OK | Warning | Error | Unknown - Success bool - Latency time.Duration -} +```bash +docker build -t netdiag . +docker run --rm netdiag ping 1.1.1.1 +docker run --rm netdiag scan 192.168.1.1 -p 1-65535 --fast ``` -One type means `--json`, table rendering, exit codes, and structured logging are -all implemented once rather than per command. - -### The shared runner +The image is ~28 MB and runs as an unprivileged user. `cap_net_raw` is applied +to the binary itself, so ICMP and SYN scanning work without running the +container as root. -`cmd/run.go` owns everything that must behave identically across commands: - -| Concern | Behaviour | -| ------------------- | ------------------------------------------------------ | -| Cancellation | `signal.NotifyContext` — Ctrl+C stops a scan mid-flight | -| Error normalization | A hard error becomes a `Result` via `probe.ErrorResult` | -| Logging | One structured line per probe, always to stderr | -| `--json` | Short-circuits rendering, prints the raw `Result` | -| Color | `output.PrintBySeverity` maps severity to color | -| Exit code | Derived from `Success` and `Severity` | - -### Concurrency - -- **Port scanner** — a semaphore-bounded worker pool (`--concurrency`, default - 100), with every dial carrying the cancellable context. -- **Ping** — `errgroup` with `SetLimit`, so pinging 100 hosts does not open 100 - sockets at once. -- **Discover** — bounded sweep of the detected network, capped at 1024 - addresses so a `/16` interface cannot launch a 65k-host scan. - -### ICMP privileges - -Raw ICMP sockets need `CAP_NET_RAW` or root. Many systems also offer -*unprivileged* ICMP datagram sockets, which need neither. - -`pkg/probe/icmp.go` tries the mode most likely to work on the current platform, -transparently retries with the other on a permission error, and caches the -result for the rest of the process. If neither works, the error explains exactly -how to fix it rather than reporting a bare `socket: permission denied`. - -## 🔐 Permissions - -Only the ICMP-based commands (`ping`, `trace`, `discover`) need special -permissions. `scan`, `http`, `dig`, `whois`, and `speedtest` never do. - -### Linux - -`ping` and `discover` first try unprivileged ICMP datagram sockets, so on many -systems they work with no setup at all. If your kernel does not allow them, -pick one of: +For a hardened deployment, drop everything and add back the one capability: ```bash -# Preferred: grant only the capability this binary needs -sudo setcap cap_net_raw+ep /usr/local/bin/netdiag - -# Or: allow unprivileged ICMP for all users -sudo sysctl -w net.ipv4.ping_group_range="0 2147483647" +docker run --rm --cap-drop=ALL --cap-add=NET_RAW netdiag scan host --fast ``` -`make install` applies the `setcap` step for you. netdiag prints whichever of -these applies if it hits a permission error. - -`trace` always needs raw sockets, so it requires `cap_net_raw` or `sudo`. +`--cap-drop=ALL` on its own will not start the image: Linux refuses to exec a +binary carrying permitted capabilities the process could never be granted, so +you get an exec error rather than a fallback to unprivileged scanning. -### macOS +## Architecture -Unprivileged ICMP works out of the box for `ping` and `discover`. `trace` -requires `sudo`. +```text +main.go + └── cmd/ Cobra commands: flags, validation, rendering + └── run.go runProbe(): cancellation, logging, --json, exit codes + │ + ▼ + pkg/probe/ all network logic behind the Prober interface + │ + ▼ + pkg/output/ tables, color, JSON + pkg/logger/ log/slog, stderr by default + pkg/config/ Viper loader for ~/.netdiag.yaml +``` -### Windows +`cmd/` imports `pkg/probe/`, never the reverse. Every probe implements the same +two-method interface and returns a typed `Result`; one shared runner turns that +into output, logs, and an exit code. -Run Command Prompt or PowerShell as Administrator for the ICMP commands. +[docs/architecture.md](docs/architecture.md) has the full picture, including a +diagram and the reasoning behind the `Prober`/`Result` design. ---- +## Responsible use -## 🤝 Contributing +`netdiag scan` and `netdiag discover` send unsolicited traffic to hosts. +Scanning or sweeping systems you do not own, or do not have explicit written +permission to test, is unlawful in many jurisdictions. Use them on your own +infrastructure or with documented authorization. -Contributions are welcome! Here are some ideas for enhancements: +## Contributing -- [ ] MTR (My Traceroute) implementation for continuous latency monitoring -- [ ] IP geolocation lookup -- [ ] mDNS/Zeroconf service discovery -- [ ] Full IPv6 support across all commands (`dig AAAA` is done) -- [ ] Packet capture / PCAP export +Contributions welcome. Ideas that would fit: -Larger planned work is tracked in [ROADMAP.md](ROADMAP.md). +- MTR-style continuous latency monitoring +- IP geolocation lookup +- mDNS/Zeroconf service discovery +- Full IPv6 support across all commands (`dig AAAA` is done) +- Packet capture / PCAP export -### Development Setup +Scope decisions, including the phases that were deliberately cut, are in +[ROADMAP.md](ROADMAP.md). ```bash -# Clone the repo git clone https://github.com/ARCoder181105/netdiag.git cd netdiag - -# Install dependencies go mod download - -# Run tests -go test ./... - -# Build -go build -o netdiag +make test # go test ./... +make lint # golangci-lint +make fmt # gofumpt + gci +make build ``` -### Submitting Changes - -1. Fork the repository -2. Create a feature branch (`git checkout -b feature/amazing-feature`) -3. Commit your changes (`git commit -m 'Add amazing feature'`) -4. Push to the branch (`git push origin feature/amazing-feature`) -5. Open a Pull Request - ---- - -## 📄 License - -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. - ---- - -## 🙏 Acknowledgments - -Built with these excellent Go libraries: - -- [Cobra](https://github.com/spf13/cobra) - CLI framework -- [pro-bing](https://github.com/prometheus-community/pro-bing) - ICMP operations -- [tablewriter](https://github.com/olekukonko/tablewriter) - Table formatting -- [color](https://github.com/fatih/color) - Terminal colors -- [speedtest-go](https://github.com/showwin/speedtest-go) - Speed testing -- [whois](https://github.com/likexian/whois-go) - WHOIS queries - ---- - -## 📞 Support +Then fork, branch, and open a pull request. +[CONTRIBUTING.md](CONTRIBUTING.md) has the details. -For issues, questions, or feature requests, please [open an issue](https://github.com/ARCoder181105/netdiag/issues). +## License ---- +MIT — see [LICENSE](LICENSE). -**Made with ❤️ by ARCoder181105** +Built with [Cobra](https://github.com/spf13/cobra), +[pro-bing](https://github.com/prometheus-community/pro-bing), +[gopacket](https://github.com/gopacket/gopacket), +[tablewriter](https://github.com/olekukonko/tablewriter), +[color](https://github.com/fatih/color), +[speedtest-go](https://github.com/showwin/speedtest-go), and +[whois](https://github.com/likexian/whois). diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..d8bae43 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,190 @@ +# Architecture + +netdiag is a CLI with eight subcommands that do very different things — ICMP +echo, TCP port scanning, DNS queries, HTTP requests, WHOIS lookups. The design +problem is that users expect them to behave *identically* in every way that is +not about the network: the same JSON shape, the same exit codes, the same +response to Ctrl+C, the same place logs go. + +The answer is a single interface, a single result type, and a single execution +path that every command routes through. + +## Layout + +```mermaid +flowchart TD + main["main.go
version wiring"] --> root + + subgraph cmd["cmd/ — thin Cobra wrappers"] + root["root.go
global flags, logger + config init"] + cmds["ping.go, scan.go, trace.go, http.go,
dig.go, whois.go, discover.go, speedtest.go
flag parsing and rendering only"] + run["run.go — runProbe()
the shared execution path"] + root --> cmds --> run + end + + subgraph probe["pkg/probe/ — all network logic"] + iface["types.go
Prober interface, Result, Severity"] + impls["PingProber, ConnectScanner, SYNScanner,
TraceProber, HTTPProber, DigProber,
DiscoverProber, WhoisProber, SpeedTestProber"] + icmp["icmp.go
privileged/unprivileged ICMP negotiation"] + iface --- impls --- icmp + end + + run -->|"Probe(ctx) → Result"| probe + + subgraph out["presentation and support"] + output["pkg/output/
tables, color, JSON"] + logger["pkg/logger/
log/slog, stderr by default"] + config["pkg/config/
Viper, ~/.netdiag.yaml"] + end + + run --> output + run --> logger + root --> config + + classDef pkg fill:#0d1117,stroke:#30363d,color:#c9d1d9 + class cmd,probe,out pkg +``` + +The dependency direction is strict: `cmd/` imports `pkg/probe/`, never the +reverse. A probe cannot reach the terminal even by accident. + +## The `Prober` contract + +Every probe implements two methods: run against a context, and report its own +type name. That is the entire interface. Adding a probe means writing one file +in `pkg/probe/` and one thin file in `cmd/` — no registry, no plugin system, no +initialization order to get wrong. + +The interface is deliberately narrow enough that `cmd/run.go` can be the only +consumer of it. When there is exactly one caller, behavior cannot drift between +commands, because there is no second place for it to drift to. + +## Why probes never print + +A probe returns a `Result` and prints nothing. This is the rule the rest of the +design hangs off, and it buys four things at once: + +- **`--json` works everywhere for free.** There is no per-command JSON + serialization to keep in sync with per-command table rendering, because the + thing being serialized is the same value the renderer receives. +- **Probes are testable without capturing stdout.** Tests assert on a returned + struct. +- **stdout stays clean.** Every diagnostic — logs, fallback notices, usage + errors — goes to stderr, so `netdiag scan host --json | jq` never chokes on a + warning that got mixed into the output. +- **One rendering path per format.** Color and table layout live in + `pkg/output/`, used by `cmd/`, and nowhere else. + +Where a probe genuinely needs to tell the user something mid-run — the SYN +scanner falling back to a connect scan, for instance — it takes a `Notify +func(string)` callback that the command points at stderr. The probe still does +not know what a terminal is. + +## The `Result` envelope + +One struct carries every probe's output: identity fields, an outcome +(`Success`, `Severity`, `Message`, `Latency`), and at most one non-nil +probe-specific payload pointer. A ping fills `PingData`, a scan fills +`ScanData`, and the rest stay nil and are omitted from JSON. A probe that could +not run at all fills none of them — `ErrorResult` carries the outcome and no +payload — so consumers must treat the payload as optional rather than assume the +one matching `probe_type` is present. + +This is a tagged union expressed with pointers rather than an interface, chosen +because it marshals to predictable JSON with no custom marshaler. The cost is +that `Result` grows a field per probe type; the benefit is that JSON consumers +see a stable, self-describing shape and the renderer can switch on which payload +is present. + +## Severity and the exit-code contract + +`Severity` is the probe's judgment about the target — not about whether the +probe worked. That distinction is what makes the exit codes useful in scripts: + +| Code | Meaning | +|---|---| +| 0 | Ran; target is healthy **or** degraded-but-alive (`Warning`) | +| 1 | Ran; target failed (`Error`) | +| 2 | Bad arguments, flags, or configuration | +| 3 | Could not run — no privileges, unresolvable host | + +**Warnings exit 0 on purpose.** A certificate with 10 days left, 25% packet loss +on a host that still answers, a scan interrupted part-way: all real conditions +worth reporting, none of them a reason to break `netdiag http api.example.com && +./deploy.sh`. Scripts that want to be stricter read `.severity` from the JSON. + +Inside a probe the same split applies. Returning `(Result{Severity: Error}, +nil)` means "I ran; the target is bad." Returning an `error` means "I could not +run" — and `runProbe` turns that into an `ErrorResult` and exit 3. Getting this +backwards is the easiest way to make a scan of an unreachable host look like a +scan that found nothing. + +## The shared runner + +`cmd/run.go` owns every behavior that must not vary between commands: + +| Concern | Where it is handled | +|---|---| +| Cancellation | `signal.NotifyContext` — Ctrl+C cancels the probe's context mid-flight | +| Error normalization | A returned error becomes a `Result` via `probe.ErrorResult` | +| Structured logging | One line per probe, always stderr or `--log-file` | +| `--json` | Short-circuits rendering and prints the `Result` | +| Color | `output.PrintBySeverity` maps severity to color | +| Exit code | Derived from `Success` and `Severity` | + +`ping` is the one command that does not call `runProbe` directly, because it +targets several hosts at once. It reuses the same helpers (`logResult`, +`exitForAll`) rather than reimplementing them. + +## ICMP privilege negotiation + +Raw ICMP sockets need `CAP_NET_RAW` or root. Most systems also offer +*unprivileged* ICMP datagram sockets, which need neither — but Windows has none, +and Linux gates them behind a sysctl that is empty by default on some +distributions. + +`pkg/probe/icmp.go` picks the mode most likely to work from `GOOS` and the +effective uid, retries with the other mode on a permission error, and caches +whichever worked for the rest of the process — so a `discover` sweep pinging +1,024 hosts pays the fallback cost at most once. When neither mode is available, +the error tells the user the exact command to fix it instead of surfacing a bare +`socket: permission denied`. + +This pattern — try, detect the permission failure specifically, degrade rather +than fail — is reused by the SYN scanner. + +## The SYN scanner + +`--fast` replaces the connect scan's full TCP handshake with a half-open probe: +send a SYN, read the reply, never complete the connection. SYN-ACK means open, +RST means closed, silence means filtered. + +Three things are worth pointing at: + +- **The checksum is computed here, not by the packet library.** `gopacket` lays + out and decodes the TCP header, but the checksum over the IPv4 pseudo-header + is netdiag's own code, tested against the RFC 1071 worked example and a + known-good vector generated independently. +- **Replies are correlated, not counted.** A raw socket receives *every* TCP + segment on the machine, including this process's own outbound SYNs and the + kernel's RSTs. A reply is only accepted if it arrives on the scan's source + port and acknowledges the exact sequence number sent to that port. +- **It degrades instead of failing.** No `CAP_NET_RAW`, or no route-derived + source address, and the scan falls back to the connect scanner with a one-line + notice on stderr. + +Measured performance, and the two bugs that benchmarking found, are in +[performance.md](performance.md). + +## Concurrency + +- **Connect scan** — semaphore-bounded worker pool, one goroutine per port, + every dial carrying the cancellable context. +- **SYN scan** — one goroutine paces sends and owns the in-flight queue without + locks; a pool of workers does nothing but write packets, from separate sockets + because the kernel serializes writes per socket. Concurrency adapts by + additive-increase/multiplicative-decrease. +- **Ping** — `errgroup` with `SetLimit`, so pinging 100 hosts does not open 100 + sockets at once. +- **Discover** — bounded sweep capped at 1,024 addresses, so a `/16` interface + cannot launch a 65,000-host scan by accident. diff --git a/docs/demo.gif b/docs/demo.gif new file mode 100644 index 0000000..42caf4e Binary files /dev/null and b/docs/demo.gif differ diff --git a/docs/demo.tape b/docs/demo.tape new file mode 100644 index 0000000..4d0ac9f --- /dev/null +++ b/docs/demo.tape @@ -0,0 +1,79 @@ +# netdiag demo recording. +# +# Produces docs/demo.gif, which the README references. +# +# Linux only, because of the setcap step: +# +# go build -o /tmp/netdiag . && sudo setcap cap_net_raw+ep /tmp/netdiag +# vhs docs/demo.tape +# +# On macOS, record with sudo instead. The commands themselves are portable. +# +# Grant the capability before recording. Without it, `scan --fast` falls back to +# a connect scan and prints the fallback notice, so the frame meant to show the +# SYN scanner would show it not being used. (`ping` would likely still work, +# since it falls back to unprivileged ICMP datagram sockets on most Linux +# systems, and `trace` would fail outright — but neither is what this recording +# is for.) +# +# vhs: https://github.com/charmbracelet/vhs + +Output docs/demo.gif + +Set Shell bash +Set FontSize 18 +Set Width 1500 +Set Height 800 +Set Padding 30 +Set Theme "Catppuccin Mocha" +Set TypingSpeed 55ms +Set PlaybackSpeed 1.0 +Set CursorBlink false +Set WindowBar "Colorful" + +# Use the capability-granted build for the recording, and keep the prompt short +# so the commands are what stands out. +Hide +Type "export PATH=/tmp:$PATH; PS1='$ '; clear" Enter +Show + +Sleep 500ms + +# ── Ping several hosts at once ─────────────────────────────────────────────── +# Two hosts, 3 packets each at a 1s interval, measured at ~2.2s locally. The +# sleep clears the 5s per-run timeout instead, so a slow or unreachable host +# cannot leave the next keystrokes landing mid-command. +Type "netdiag ping google.com cloudflare.com" Sleep 400ms Enter +Sleep 6s + +Type "clear" Enter Sleep 300ms + +# ── Half-open SYN scan ─────────────────────────────────────────────────────── +Type "netdiag scan 127.0.0.1 -p 1-1024 --fast" Sleep 400ms Enter +Sleep 3s + +Type "clear" Enter Sleep 300ms + +# ── Both scan methods, measured against the same target ────────────────────── +Type "netdiag scan 127.0.0.1 -p 1-65535 --benchmark" Sleep 400ms Enter +Sleep 5s + +Type "clear" Enter Sleep 300ms + +# ── TLS check, and the JSON contract behind every command ──────────────────── +Type "netdiag http https://example.com" Sleep 400ms Enter +Sleep 3s + +Type "netdiag http https://example.com --json | jq '{status: .http_data.status_code, tls_days: .http_data.tls_days_left}'" Sleep 400ms Enter +Sleep 3s + +Type "clear" Enter Sleep 300ms + +# ── Exit codes are the scripting contract ──────────────────────────────────── +Type "netdiag dig github.com MX" Sleep 400ms Enter +Sleep 3s + +Type 'echo "exit: $?"' Sleep 300ms Enter +Sleep 2s + +Sleep 1s