From 86fbef5dbb57f91af670a9982fe74deb2933fcb0 Mon Sep 17 00:00:00 2001 From: d3vhex Date: Fri, 4 Sep 2026 00:09:30 +0300 Subject: [PATCH] Say where the telemetry stops, instead of showing an empty table Techniques compare through the ATT&CK hierarchy rather than as strings, the console gets a design system and charts, and four separate silent breaks behind one blank inventory tab are fixed and pinned: a snapshot table that was also deduplicated, an encrypted column too narrow for its ciphertext, a collector chain where the first failure disabled the four after it, and a read that never decrypted. --- Sentora/build_agent.ps1 | 1 + Sentora/build_agent.sh | 1 + Sentora/main.py | 49 ++- Sentora/modules/edr_enforcer.py | 124 +++++--- Sentora/modules/telemetry_health.py | 105 +++++++ app.py | 366 +++++++++++++++++++++- core/attack.py | 180 +++++++++++ db/init.sql | 8 +- frontend/package-lock.json | 407 ++++++++++++++++++++++++- frontend/package.json | 3 +- frontend/src/App.tsx | 20 ++ frontend/src/components/ui/charts.tsx | 156 ++++++++++ frontend/src/components/ui/index.tsx | 322 +++++++++++++++++++ frontend/src/index.css | 141 +++++++-- frontend/src/pages/AgentDetail.tsx | 16 + frontend/src/pages/Assets.tsx | 22 +- frontend/src/pages/AttackChain.tsx | 189 ++++++++++++ frontend/src/pages/AttackCoverage.tsx | 57 +++- frontend/src/pages/Dashboard.tsx | 147 +++++++-- frontend/src/pages/TelemetryHealth.tsx | 181 +++++++++++ frontend/src/pages/ThreatIntel.tsx | 15 +- server.py | 114 ++++++- tests/test_attack_hierarchy.py | 210 +++++++++++++ tests/test_collector_isolation.py | 149 +++++++++ tests/test_design_system.py | 241 +++++++++++++++ tests/test_encrypted_column_widths.py | 160 ++++++++++ tests/test_ingest_modes.py | 131 ++++++++ tests/test_telemetry_health.py | 222 ++++++++++++++ 28 files changed, 3594 insertions(+), 143 deletions(-) create mode 100644 Sentora/modules/telemetry_health.py create mode 100644 core/attack.py create mode 100644 frontend/src/components/ui/charts.tsx create mode 100644 frontend/src/components/ui/index.tsx create mode 100644 frontend/src/pages/AttackChain.tsx create mode 100644 frontend/src/pages/TelemetryHealth.tsx create mode 100644 tests/test_attack_hierarchy.py create mode 100644 tests/test_collector_isolation.py create mode 100644 tests/test_design_system.py create mode 100644 tests/test_encrypted_column_widths.py create mode 100644 tests/test_ingest_modes.py create mode 100644 tests/test_telemetry_health.py diff --git a/Sentora/build_agent.ps1 b/Sentora/build_agent.ps1 index bbc7700..154563a 100644 --- a/Sentora/build_agent.ps1 +++ b/Sentora/build_agent.ps1 @@ -182,6 +182,7 @@ $hiddenImports = @( # builds cleanly and dies on the first channel connect, which is the worst # place to find it. "modules.version", + "modules.telemetry_health", "websocket", "winpty", "winpty.ptyprocess", diff --git a/Sentora/build_agent.sh b/Sentora/build_agent.sh index 39a4b18..5b8539f 100644 --- a/Sentora/build_agent.sh +++ b/Sentora/build_agent.sh @@ -205,6 +205,7 @@ step "Running PyInstaller..." --hidden-import modules.console \ --hidden-import modules.link \ --hidden-import modules.version \ + --hidden-import modules.telemetry_health \ --hidden-import websocket \ --hidden-import modules.db \ --exclude-module torch \ diff --git a/Sentora/main.py b/Sentora/main.py index 32dff88..1420188 100644 --- a/Sentora/main.py +++ b/Sentora/main.py @@ -22,6 +22,7 @@ import modules.agent_paths as agent_paths import modules.console as console import modules.link as link +import modules.telemetry_health as telemetry_health import logging import builtins @@ -539,7 +540,16 @@ def send_table(table: str): s.sendall(struct.pack('!Q', fsize)) s.sendall(data_bytes) + # Marked sent because `sendall` returned, which means the bytes + # reached the socket buffer. The ingest protocol has no reply, so + # this is the strongest thing the agent can know - and it is weaker + # than it reads. A server that accepts a batch and stores none of it + # looks identical from here, which is exactly how four tables came to + # be permanently empty while this line said "sent (50 rows)" every + # cycle. `modules/telemetry_health` exists so the server can put its + # own row count beside this number. mark_sent(table, [r['id'] for r in rows]) + telemetry_health.record_send(table, len(rows)) if debug: # The IP/OS/HOST/MAC banner used to be repeated here. It is # constant for the life of the process and was the single largest @@ -548,6 +558,7 @@ def send_table(table: str): print(f"[+] {table} sent ({len(rows)} rows)") except Exception as e: + telemetry_health.record_send_failure(table, e) if debug: print(f"[!] Sending error ({table}): {e}") @@ -563,13 +574,40 @@ def db_sender_loop(): time.sleep(10) +#: How many consecutive failures between repeat reports, once a collector has +#: announced itself broken. Loud on the first, then rare - a collector that +#: has been failing for a day should not be writing a line every two minutes. +_FAILURE_REPEAT_EVERY = 30 + + def periodic_wrapped(func, interval: int, name: str): + """Run a collector forever, and say when it stops working. + + The failure report used to be behind `if debug`, which nothing sets. A + collector that raised on its first line failed silently every interval + for the life of the agent, and the only symptom was an empty table in the + console - indistinguishable from a host that genuinely has nothing to + report. Hardware and network inventory were both empty for exactly this + reason, and nothing anywhere said so. + + Rate-limited rather than unconditional, because the same argument that + makes silence wrong makes a line every two minutes useless: it would bury + the schema errors and the auth rejections that also live in this log. + """ + failures = 0 while True: try: func() + if failures: + print(f"[+] {name} recovered after {failures} failure(s)", flush=True) + failures = 0 except Exception as e: - if debug: - print(f"[!] Error: ({name}): {e}") + failures += 1 + if failures == 1 or failures % _FAILURE_REPEAT_EVERY == 0: + print(f"[!] {name} failed ({failures} consecutive): " + f"{type(e).__name__}: {e}", flush=True) + if debug: + traceback.print_exc() time.sleep(interval) @@ -1924,6 +1962,13 @@ def dispatch_channel_request(method: str, path: str, body) -> tuple[dict, int]: if path == "/health": return {"ok": True, "agent": AGENT_NAME}, 200 + if path == "/telemetry/health": + # What this agent holds and believes it has shipped, per table. The + # server puts its own row counts beside these; neither half alone can + # tell a quiet host from a broken pipeline. + return {"ok": True, "agent": AGENT_NAME, + "tables": telemetry_health.report(TABLES)}, 200 + # Named rather than a bare 404: a path the agent does not implement is # usually a server that is newer than this build, and saying so is the # difference between "upgrade the agent" and an afternoon of guessing. diff --git a/Sentora/modules/edr_enforcer.py b/Sentora/modules/edr_enforcer.py index b84e769..7301648 100644 --- a/Sentora/modules/edr_enforcer.py +++ b/Sentora/modules/edr_enforcer.py @@ -69,12 +69,23 @@ def check_fim(): }) def track_network(): + """Established outbound connections, with the process behind each. + + `psutil.net_connections` raises AccessDenied where the agent is not + privileged enough to see other processes' sockets. That is a real + condition worth reporting rather than an empty table: the caller isolates + and logs it, and an operator can act on "the agent cannot read the + connection table" in a way they cannot act on a blank page. + """ for conn in psutil.net_connections(kind='inet'): if conn.status == 'ESTABLISHED' and conn.raddr: try: proc = psutil.Process(conn.pid) name = proc.name() except Exception: + # The process ended between listing and lookup, which is + # ordinary on a busy host. The connection is still worth + # recording. name = "Unknown" insert_record_enc("network_connections", { @@ -150,53 +161,90 @@ def monitor_processes(): continue def get_hardware_inventory(): + # Nothing is swallowed here any more. An empty hardware tab is + # indistinguishable from a host with no hardware, so a collector that + # cannot run has to say why - the console cannot infer it. if IS_WINDOWS: ps_cmd = "[Console]::OutputEncoding = [System.Text.Encoding]::UTF8; Get-PnpDevice -PresentOnly | Select-Object FriendlyName, InstanceId | ConvertTo-Json" - try: - res = subprocess.run(["powershell", "-NoProfile", "-Command", ps_cmd], capture_output=True, text=True, encoding='utf-8', errors='replace') - if res.returncode == 0 and res.stdout: - data = json.loads(res.stdout) - for item in (data if isinstance(data, list) else [data]): - if not item or not item.get("FriendlyName"): continue - insert_record_enc("hardware_inventory", { - "type": "pnp", - "name": item.get("FriendlyName"), - "product_id": item.get("InstanceId", "N/A"), - "serial_number": item.get("InstanceId", "N/A"), - "status": "active" - }) - except Exception: pass + res = subprocess.run(["powershell", "-NoProfile", "-Command", ps_cmd], capture_output=True, text=True, encoding='utf-8', errors='replace') + if res.returncode != 0: + raise RuntimeError( + f"Get-PnpDevice exited {res.returncode}: " + f"{(res.stderr or '').strip()[:200]}") + if not (res.stdout or "").strip(): + raise RuntimeError("Get-PnpDevice returned nothing") + data = json.loads(res.stdout) + for item in (data if isinstance(data, list) else [data]): + if not item or not item.get("FriendlyName"): + continue + insert_record_enc("hardware_inventory", { + "type": "pnp", + "name": item.get("FriendlyName"), + "product_id": item.get("InstanceId", "N/A"), + "serial_number": item.get("InstanceId", "N/A"), + "status": "active" + }) else: - try: - res = subprocess.run(["lsusb"], capture_output=True, text=True, encoding='utf-8', errors='replace') - for line in res.stdout.splitlines(): + res = subprocess.run(["lsusb"], capture_output=True, text=True, encoding='utf-8', errors='replace') + if res.returncode != 0: + raise RuntimeError( + f"lsusb exited {res.returncode}: " + f"{(res.stderr or '').strip()[:200]}") + for line in res.stdout.splitlines(): + if line.strip(): insert_record_enc("hardware_inventory", { "type": "usb", "name": line.strip() }) - except Exception: pass def monitor_registry(): - if not IS_WINDOWS: return + if not IS_WINDOWS: + return ps_cmd = "Get-ItemProperty HKLM:\\Software\\Microsoft\\Windows\\CurrentVersion\\Run | ConvertTo-Json" - try: - res = subprocess.run(["powershell", "-Command", ps_cmd], capture_output=True, text=True, encoding='utf-8', errors='replace') - if res.returncode == 0 and res.stdout: - data = json.loads(res.stdout) - for k, v in data.items(): - if k not in ["PSPath", "PSParentPath", "PSChildName", "PSDrive", "PSProvider"]: - insert_record_enc("registry_logs", { - "hive": "HKLM", - "key_path": "Software\\Microsoft\\Windows\\CurrentVersion\\Run", - "value_name": k, - "value_data": str(v), - "status": "monitored" - }) - except Exception: pass + res = subprocess.run(["powershell", "-NoProfile", "-Command", ps_cmd], capture_output=True, text=True, encoding='utf-8', errors='replace') + if res.returncode != 0: + raise RuntimeError( + f"Get-ItemProperty exited {res.returncode}: " + f"{(res.stderr or '').strip()[:200]}") + if not (res.stdout or "").strip(): + return # an empty Run key is a real answer + data = json.loads(res.stdout) + for k, v in data.items(): + if k not in ["PSPath", "PSParentPath", "PSChildName", "PSDrive", "PSProvider"]: + insert_record_enc("registry_logs", { + "hive": "HKLM", + "key_path": "Software\\Microsoft\\Windows\\CurrentVersion\\Run", + "value_name": k, + "value_data": str(v), + "status": "monitored" + }) def main(): - check_fim() - track_network() - monitor_processes() - get_hardware_inventory() - monitor_registry() + """Every collector, each isolated from the others. + + They used to be five bare calls in a row, so the first one to raise took + the rest of the cycle with it - and since the caller retries on a timer, + it took them again every two minutes, permanently. + + That is why the console showed no hardware and no network inventory at + the same time: `track_network` and `get_hardware_inventory` sit second and + fourth, and something ahead of them was throwing on every pass. An empty + table reads as "this host has nothing to report", which is the one thing + it did not mean. + + One failing collector now costs its own data and nothing else, and says + so. `periodic_wrapped` rate-limits the repeat, so a permanently broken + collector is loud once rather than every two minutes forever. + """ + for name, collector in ( + ("fim", check_fim), + ("network", track_network), + ("processes", monitor_processes), + ("hardware", get_hardware_inventory), + ("registry", monitor_registry), + ): + try: + collector() + except Exception as e: + print(f"[edr] {name} collector failed: {type(e).__name__}: {e}", + flush=True) diff --git a/Sentora/modules/telemetry_health.py b/Sentora/modules/telemetry_health.py new file mode 100644 index 0000000..8bf875c --- /dev/null +++ b/Sentora/modules/telemetry_health.py @@ -0,0 +1,105 @@ +"""What this agent has collected, and what it believes it has shipped. + +Three bugs in one day had the same shape: every layer reported success and +the chain was broken anyway, and the only symptom was an empty table - +indistinguishable from a host that genuinely has nothing to report. + + a collector raised on its first line, so the four after it never ran + the server accepted a batch and discarded every row of it + the agent marked rows sent as soon as `sendall` returned + +The last one is why none of it was visible from here. `sendall` returning +means the bytes reached the operating system's socket buffer. It says nothing +about whether the server stored them, and the ingest protocol has no reply - +so the agent cannot know, and has always reported success. + +This module does not fix that. It makes the two halves *comparable*: the agent +says what it holds and what it has shipped, the server says what it holds, and +the difference names the broken link. `[+] network_connections sent (50 rows)` +against a server table with zero rows is not ambiguous once somebody puts the +two numbers next to each other - the whole difficulty was that nobody ever did. + +Counts are read from the database rather than kept as counters. A counter +resets when the agent restarts and drifts whenever anything writes without +going through it; the tables are the truth, and reading them costs two +queries per cycle. +""" + +from __future__ import annotations + +import threading +import time + +#: Per table: when the last send succeeded, how many rows it carried, and the +#: last error if there was one. Small, and the only thing here that is not +#: read straight from the database. +_send_state: dict[str, dict] = {} +_lock = threading.Lock() + + +def record_send(table: str, rows: int) -> None: + with _lock: + state = _send_state.setdefault(table, {}) + state["last_sent_at"] = time.time() + state["last_sent_rows"] = int(rows) + state["last_error"] = None + + +def record_send_failure(table: str, error: Exception | str) -> None: + """A send that raised. Kept rather than logged only, because the console + asking "why is this table empty" needs the answer, and the answer is on + the endpoint.""" + with _lock: + state = _send_state.setdefault(table, {}) + state["last_error"] = f"{type(error).__name__}: {error}" \ + if isinstance(error, Exception) else str(error) + state["last_error_at"] = time.time() + + +def _counts(table: str) -> tuple[int, int]: + """(rows held, rows not yet shipped) for one table. + + A table that does not exist is `(0, 0)` and not an error: the agent's + schema gains tables over releases, and a missing one is a real state the + report should show rather than an exception that hides every other table + behind it. + """ + from modules.db import get_conn + + try: + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute(f"SELECT COUNT(*), COUNT(*) FILTER (WHERE sent = FALSE) " + f"FROM {table}") + row = cur.fetchone() + return int(row[0] or 0), int(row[1] or 0) + except Exception: + return 0, 0 + + +def report(tables) -> dict: + """Everything this agent knows about its own telemetry, per table. + + Deliberately flat and dumb. The judgement about what the numbers *mean* + lives on the server, where the other half of the comparison is - putting + it here would mean an agent deciding whether its own data arrived. + """ + with _lock: + state = {t: dict(v) for t, v in _send_state.items()} + + out = {} + for table in tables: + held, unsent = _counts(table) + sent = state.get(table, {}) + out[table] = { + "held": held, + "unsent": unsent, + # Rows this agent believes it has shipped. Belief, not fact: + # nothing acknowledges an ingest batch. + "shipped": max(held - unsent, 0), + "last_sent_at": sent.get("last_sent_at"), + "last_sent_rows": sent.get("last_sent_rows"), + "last_error": sent.get("last_error"), + "last_error_at": sent.get("last_error_at"), + } + return out diff --git a/app.py b/app.py index 2f6062d..0622e43 100644 --- a/app.py +++ b/app.py @@ -111,6 +111,7 @@ def password_complexity(cls, v: str): from security import session as session_store from security import ssrf from core import agent_link +from core import attack from core import config_validation from core import login_guard from core import netloc @@ -2543,7 +2544,14 @@ async def get_software_inventory(request, agent): @require_permission("read_telemetry") @app.route("/api/agent//inventory/network") async def get_network_inventory(request, agent): - return await stream_from_db_dec("network_connections", agent, connect_db_for_agent) + # The fields were missing here and nowhere else, so the Network tab + # rendered `enc::gAAAAAB...` where the process name should be. Ciphertext + # in a column reads as data, not as a fault - the page looked populated + # and was unusable. `test_encrypted_reads_decrypt` now requires every + # route reading an encrypted table to pass its fields. + fields = ENCRYPTED_FIELDS_MAP.get("network_connections") + return await stream_from_db_dec("network_connections", agent, + connect_db_for_agent, encrypted_fields=fields) @require_permission("read_telemetry") @app.route("/api/agent//fim") @@ -6192,6 +6200,12 @@ async def get_agent_details(name): detail["agent_version"] = reported detail["version_state"] = state detail["version_detail"] = why + # Whether a command would reach it *right now*, which is not the + # same question as whether it is reporting. Telemetry and control + # travel on different connections, so an agent can be sending data + # and be uncommandable - and the console has been unable to say + # which since the fallback went. + detail["channel_connected"] = _linked_agent(detail["name"]) is not None return sanic_json({"agents": agent_details, "server_version": product_version.SERVER_VERSION, @@ -6924,6 +6938,15 @@ async def get_attack_coverage(request): observed = {} covered = sorted(_sigma_technique_index()) + + # Compared through the technique hierarchy, not as strings. + # + # This was `set(covered) - set(observed)` both ways, so a rule tagged + # `T1003.001` and an event carrying `T1003` never matched each other: one + # landed in "covered but never seen" and the other in "seen with nothing + # covering it", and both were wrong. Those two lists are the only ones an + # operator acts on. + states = attack.classify(covered, observed) return sanic_json({ "status": "success", "covered": covered, @@ -6931,12 +6954,22 @@ async def get_attack_coverage(request): for k, v in sorted(observed.items(), key=lambda kv: -kv[1])], "covered_count": len(covered), "observed_count": len(observed), - # Covered but never seen. Normal, and worth showing separately from - # the third category below. - "quiet": sorted(set(covered) - set(observed)), - # Seen with no rule that covers it - a technique the AI or the regex - # list surfaced and Sigma has nothing for. - "uncovered_but_seen": sorted(set(observed) - set(covered)), + # Covered and never seen. Normal, and the healthy majority. + "quiet": attack.unseen(covered, observed), + # Seen with nothing addressing it at all. + "uncovered_but_seen": states["none"], + # Seen, with rules for *other* sub-techniques of the same parent and + # not this one. Reported separately because it is the state most + # easily mistaken for coverage - the parent cell is green while this + # particular action would go unnoticed - and because the fix differs: + # `none` needs a rule written, this usually needs one widened. + "covered_only_by_a_sibling": states["sibling"], + # Seen at sub-technique granularity, with a rule that claims the whole + # parent. + "covered_by_the_parent": states["parent"], + # What the grid rolls up, so a cell can say "3 sub-techniques" rather + # than implying the whole technique. + "subtechniques": attack.rollup(covered), }) @@ -6978,6 +7011,325 @@ def _observed_techniques(agent: str | None = None) -> dict: return seen +#: What the server holds for a table, so the two halves can be compared. +def _server_table_counts(agent: str, tables) -> dict: + """Row count and newest row per table, from this agent's database.""" + counts: dict[str, dict] = {} + db = _agent_name_forms(agent)[1] + "_db" + with sync_mysql_conn() as conn: + cursor = conn.cursor() + try: + for table in tables: + try: + cursor.execute( + f"SELECT COUNT(*) FROM {_quote_identifier(db)}" + f".{_quote_identifier(table)}") + counts[table] = {"rows": int(cursor.fetchone()[0] or 0)} + except Exception: + # A table this server does not have. A real state - the + # agent may be newer - and not a reason to fail the rest. + counts[table] = {"rows": None} + finally: + cursor.close() + return counts + + +def _classify_link(agent_side: dict, server_rows) -> tuple[str, str]: + """Where the chain is broken for one table, if it is. + + The whole point of this file. Each of these states looked identical from + the console before - an empty table - and each needs a different person to + do a different thing. + """ + held = int(agent_side.get("held") or 0) + unsent = int(agent_side.get("unsent") or 0) + shipped = int(agent_side.get("shipped") or 0) + error = agent_side.get("last_error") + + if server_rows is None: + return ("no table", "this server has no such table; the agent may be " + "newer than it") + if held == 0: + return ("not collected", + "the agent has never written a row of this locally - the " + "collector is not running, failed, or does not apply to this host") + if error: + return ("send failing", f"the agent could not ship it: {error}") + if shipped == 0 and unsent > 0: + return ("queued", f"{unsent} row(s) collected and none shipped yet") + if shipped > 0 and server_rows == 0: + # The state that was invisible. The agent's log says "sent (50 rows)" + # every cycle and the server holds none of them. + return ("lost in transit", + f"the agent has shipped {shipped} row(s) and this server holds " + f"none - accepted and discarded, not a quiet host") + if server_rows == 0: + return ("empty", "nothing collected and nothing expected") + return ("flowing", f"{server_rows} row(s) here") + + +@app.route("/telemetry-health/") +@require_permission("read_telemetry") +async def get_telemetry_health(request, agent): + """Does this agent's telemetry actually arrive? Per table, end to end. + + Three bugs in one day had the same shape: every layer reported success, + the chain was broken anyway, and the only symptom was an empty table - + indistinguishable from a host that genuinely has nothing to report. + + A collector raised on its first line and the four after it never ran. The + server accepted a batch and discarded every row. And the agent marks rows + sent as soon as `sendall` returns, because the ingest protocol has no + reply - so it reported success in all three cases and was not wrong to. + + Neither half can tell a quiet host from a broken pipeline on its own. This + puts the two numbers next to each other, which is all it ever needed. + """ + link = _linked_agent(agent) + if link is None: + return sanic_json({"status": "error", + "message": _no_channel_message(agent)}, status=502) + try: + body, status = await link.request("GET", "/telemetry/health", timeout=15) + except agent_link.LinkError as e: + return sanic_json({"status": "error", + "message": f"the agent did not answer: {e}"}, status=502) + if status == 501: + return sanic_json({ + "status": "error", + "message": "This agent predates telemetry health reporting. " + "Rebuild and reinstall it.", + }, status=501) + + agent_tables = (body or {}).get("tables") or {} + server = await asyncio.to_thread( + _server_table_counts, agent, sorted(agent_tables)) + + report = [] + for table in sorted(agent_tables): + side = agent_tables[table] or {} + rows = server.get(table, {}).get("rows") + state, detail = _classify_link(side, rows) + report.append({ + "table": table, + "state": state, + "detail": detail, + "agent_held": side.get("held"), + "agent_unsent": side.get("unsent"), + "agent_shipped": side.get("shipped"), + "agent_last_sent_at": side.get("last_sent_at"), + "agent_last_error": side.get("last_error"), + "server_rows": rows, + }) + + broken = [r for r in report if r["state"] in + ("lost in transit", "send failing", "not collected")] + return sanic_json({ + "status": "success", + "agent": agent, + "tables": report, + "broken_count": len(broken), + }) + + +def _threat_trend(days: int = 14) -> list: + """Detections per day across the fleet, in two counts that differ. + + `detections` is every event that fired a rule. `distinct` is how many + *different* techniques were behind them, which is the number worth + watching: a thousand detections from one noisy rule and a thousand from + forty different techniques are the same bar and completely different days. + + Plotted together on two axes because the shape of one against the other is + the signal - a spike in detections with a flat distinct line is almost + always a rule that needs tuning, not an incident. + """ + buckets: dict[str, dict] = {} + with sync_mysql_conn() as conn: + cursor = conn.cursor() + try: + cursor.execute("SHOW DATABASES") + dbs = [r[0] for r in cursor.fetchall() if r[0].endswith("_db")] + for db in dbs: + try: + cursor.execute( + f"SELECT DATE(created_at) AS d, techniques, COUNT(*) " + f"FROM {_quote_identifier(db)}.siem_events " + f"WHERE created_at >= CURDATE() - INTERVAL %s DAY " + f"AND techniques IS NOT NULL AND techniques <> '' " + f"GROUP BY d, techniques", + (int(days),), + ) + except Exception: + # An agent enrolled before the column existed. Not an + # error, and not a reason to fail the whole chart. + continue + for day, value, count in cursor.fetchall(): + key = day.isoformat() if hasattr(day, "isoformat") else str(day) + bucket = buckets.setdefault( + key, {"date": key, "detections": 0, "techniques": set()}) + bucket["detections"] += int(count) + for raw in str(value).split(","): + technique = attack.normalise(raw) + if technique: + bucket["techniques"].add(technique) + finally: + cursor.close() + + # Every day in the window, including the quiet ones. A line drawn only + # through the days that had events invents a slope between them and makes + # a two-day gap look like a gradual decline. + import datetime + today = datetime.date.today() + series = [] + for offset in range(days, -1, -1): + key = (today - datetime.timedelta(days=offset)).isoformat() + bucket = buckets.get(key) + series.append({ + "date": key, + "detections": bucket["detections"] if bucket else 0, + "distinct": len(bucket["techniques"]) if bucket else 0, + }) + return series + + +@app.route("/threat-trend") +@require_permission("read_telemetry") +async def get_threat_trend(request): + """Detections per day across the fleet, for the dashboard.""" + try: + days = max(1, min(int(request.args.get("days", 14)), 90)) + except ValueError: + days = 14 + try: + series = await asyncio.to_thread(_threat_trend, days) + except Exception as e: + print(f"[!] threat trend query failed: {e}", flush=True) + return sanic_json({"status": "error", "message": str(e)}, status=500) + return sanic_json({"status": "success", "days": days, "series": series}) + + +def _observed_chain(agent: str, hours: int = 168) -> list: + """The tactics seen on one host, in kill-chain order, with when. + + A list of techniques tells you what happened. It does not tell you that + execution was followed by persistence and then credential access, which is + the difference between a suspicious command and an intrusion - and the + difference an analyst is trying to establish when they open a host. + + Ordered by the chain rather than by the clock. Timestamps are kept per + stage so the two can be read together: a chain whose stages are days apart + is a different story from one that completed inside a minute, and only the + order makes either legible. + """ + stages: dict[str, dict] = {} + db = _agent_name_forms(agent)[1] + "_db" + with sync_mysql_conn() as conn: + cursor = conn.cursor() + try: + cursor.execute( + f"SELECT techniques, MIN(created_at), MAX(created_at), COUNT(*) " + f"FROM {_quote_identifier(db)}.siem_events " + f"WHERE techniques IS NOT NULL AND techniques <> '' " + f"AND created_at >= NOW() - INTERVAL %s HOUR " + f"GROUP BY techniques", + (int(hours),), + ) + rows = cursor.fetchall() + except Exception as e: + # An agent with no telemetry, or one enrolled before the column + # existed. Not an error, and an empty chain is a true answer. + print(f"[attack-chain] {agent}: {e}", flush=True) + return [] + finally: + cursor.close() + + tactics_for = _sigma_tactics_by_technique() + for value, first, last, count in rows: + for raw in str(value).split(","): + technique = attack.normalise(raw) + if not technique: + continue + # A technique with no tactic still happened. It lands at the end + # of the chain rather than being dropped. + for tactic in tactics_for.get(technique) or ["other"]: + stage = stages.setdefault(tactic, { + "tactic": tactic, "techniques": set(), + "events": 0, "first_seen": first, "last_seen": last, + }) + stage["techniques"].add(technique) + stage["events"] += int(count) + if first and (not stage["first_seen"] or first < stage["first_seen"]): + stage["first_seen"] = first + if last and (not stage["last_seen"] or last > stage["last_seen"]): + stage["last_seen"] = last + + ordered = sorted(stages.values(), + key=lambda s: attack.tactic_position(s["tactic"])) + return [{ + "tactic": s["tactic"], + "position": attack.tactic_position(s["tactic"]), + "techniques": sorted(s["techniques"]), + "events": s["events"], + "first_seen": s["first_seen"].isoformat() if s["first_seen"] else None, + "last_seen": s["last_seen"].isoformat() if s["last_seen"] else None, + } for s in ordered] + + +def _sigma_tactics_by_technique() -> dict: + """Technique -> the tactics its rules are tagged with. + + From the rules themselves, so there is no mapping table here to drift out + of date - the same reason the coverage index reads them rather than + keeping a list. + """ + mapping: dict[str, list] = {} + try: + from core.sigma_loader import load_dir + path = os.getenv( + "SIGMA_RULES_PATH", + str(pathlib.Path(__file__).resolve().parent / + "Sentora" / "conf" / "sigma")) + for rule in load_dir(path).rules: + for technique in rule.techniques: + clean = attack.normalise(technique) + if not clean: + continue + known = mapping.setdefault(clean, []) + for tactic in rule.tactics: + if tactic not in known: + known.append(tactic) + except Exception as e: + print(f"[attack-chain] could not read rule tactics: {e}", flush=True) + return mapping + + +@app.route("/attack-chain/") +@require_permission("read_telemetry") +async def get_attack_chain(request, agent): + """What happened on this host, laid out as a kill chain.""" + try: + hours = max(1, min(int(request.args.get("hours", 168)), 24 * 90)) + except ValueError: + hours = 168 + try: + chain = await asyncio.to_thread(_observed_chain, agent, hours) + except Exception as e: + print(f"[!] attack chain query failed for {agent}: {e}", flush=True) + return sanic_json({"status": "error", "message": str(e)}, status=500) + + return sanic_json({ + "status": "success", + "agent": agent, + "hours": hours, + "chain": chain, + # The full order, so the UI can draw the stages that did *not* happen + # as gaps rather than omitting them. A chain with holes in it is the + # readable thing; a list of four tactics is not. + "tactic_order": list(attack.TACTIC_ORDER), + }) + + def _sigma_technique_index() -> set: """Techniques the agent can detect: Sigma rules plus correlation. diff --git a/core/attack.py b/core/attack.py new file mode 100644 index 0000000..127a91a --- /dev/null +++ b/core/attack.py @@ -0,0 +1,180 @@ +"""ATT&CK techniques as a hierarchy, and tactics as an order. + +Two things this exists to stop. + +**Techniques compared as opaque strings.** The coverage endpoint did +`set(covered) - set(observed)`, so a rule tagged `T1003.001` and an event +carrying `T1003` never matched: one landed in "covered but never seen", the +other in "seen with nothing covering it", and both were wrong. Those two lists +are the only ones an operator acts on. + +**Coverage claimed upwards.** Detecting `T1003.001` (LSASS memory) says +nothing about `T1003.003` (the AD database) - they are different actions with +different telemetry, and the rule for one will never fire on the other. So +covering a sub-technique gives its parent *partial* coverage and never full, +while a rule tagged with a bare parent does claim the whole thing. The +asymmetry is the point: rolled-up coverage that ignores it is how a heatmap +turns eleven green cells into a number nobody should trust. +""" + +from __future__ import annotations + +import re + +#: MITRE Enterprise, in the order an intrusion tends to move through them. +#: Used to lay a host's activity out as a chain rather than a bag of tags - +#: "we saw execution and then persistence" is a different sentence from "we +#: saw persistence and then execution", and only one of them is a foothold. +TACTIC_ORDER = ( + "reconnaissance", + "resource-development", + "initial-access", + "execution", + "persistence", + "privilege-escalation", + "defense-evasion", + "credential-access", + "discovery", + "lateral-movement", + "collection", + "command-and-control", + "exfiltration", + "impact", +) + +TACTIC_POSITION = {name: i for i, name in enumerate(TACTIC_ORDER)} + +_TECHNIQUE = re.compile(r"^T\d{4}(?:\.\d{3})?$", re.I) + + +def normalise(value: str | None) -> str: + """`" t1059.001 "` -> `"T1059.001"`, and anything else -> `""`. + + Total on purpose. Technique strings arrive from rule tags, from the AI + path, and from a regex list, and a malformed one is a thing to drop + rather than to raise on halfway through building a coverage report. + """ + text = str(value or "").strip().upper() + return text if _TECHNIQUE.match(text) else "" + + +def parent(technique: str) -> str: + """`"T1059.001"` -> `"T1059"`. A parent is its own parent.""" + clean = normalise(technique) + return clean.split(".")[0] if clean else "" + + +def is_subtechnique(technique: str) -> bool: + return "." in normalise(technique) + + +def tactic_position(tactic: str) -> int: + """Where a tactic sits in the chain; unknown tactics sort last. + + Unknown rather than dropped, because a tactic this file has not heard of + is still something that happened on the host. + """ + return TACTIC_POSITION.get(str(tactic or "").strip().lower(), len(TACTIC_ORDER)) + + +def order_tactics(tactics) -> list[str]: + """The tactics given, deduplicated and in kill-chain order.""" + seen = {t for t in (str(x or "").strip().lower() for x in tactics) if t} + return sorted(seen, key=lambda t: (tactic_position(t), t)) + + +# -------------------------------------------------------------------------- +# Coverage +# -------------------------------------------------------------------------- + +#: What a rule set says about one observed technique. +#: +#: `covered` a rule addresses exactly this +#: `parent` a rule addresses the whole parent technique, so this is +#: included in what it claims +#: `sibling` rules address other sub-techniques of the same parent and not +#: this one. NOT coverage, and the state most likely to be read as +#: coverage: the parent looks green while this specific action +#: would go unnoticed. +#: `none` nothing addresses it +COVERAGE_STATES = ("covered", "parent", "sibling", "none") + + +def coverage_of(technique: str, covered) -> str: + """How well `covered` addresses one technique. See COVERAGE_STATES.""" + target = normalise(technique) + if not target: + return "none" + + index = {normalise(c) for c in covered} + index.discard("") + if target in index: + return "covered" + + root = parent(target) + if target != root and root in index: + return "parent" + if any(c != target and parent(c) == root for c in index): + return "sibling" + return "none" + + +def classify(covered, observed) -> dict: + """Split observed techniques by how well the rule set addresses them. + + Returns one list per state in COVERAGE_STATES, each sorted. + + `sibling` is reported separately rather than folded into `none` because + the two call for different work: nothing at all needs a rule written, + while a covered sibling usually needs an existing rule widened, and + somebody reading a rolled-up heatmap would see neither. + """ + buckets: dict[str, list[str]] = {state: [] for state in COVERAGE_STATES} + for raw in observed: + clean = normalise(raw) + if clean: + buckets[coverage_of(clean, covered)].append(clean) + return {state: sorted(set(values)) for state, values in buckets.items()} + + +def unseen(covered, observed) -> list[str]: + """Techniques a rule addresses that have never fired here. + + Compared with the parent relation rather than as strings: a rule tagged + `T1003.001` has been exercised by an event carrying `T1003`, and calling + it unseen sends somebody to test a detection that is already working. + """ + seen = {normalise(o) for o in observed} + seen.discard("") + roots = {parent(o) for o in seen} + quiet = [] + for raw in covered: + clean = normalise(raw) + if not clean or clean in seen: + continue + # An observation at parent granularity counts as having exercised the + # sub-technique rules underneath it; the reverse does not hold. + if parent(clean) in seen: + continue + if clean in roots: + continue + quiet.append(clean) + return sorted(set(quiet)) + + +def rollup(techniques) -> dict: + """Parent technique -> the sub-techniques given for it. + + The grid is drawn at parent granularity because that is the only way it + fits on a screen. This keeps what was rolled up, so a cell can say "3 of + the sub-techniques here" instead of implying the whole of T1003. + """ + out: dict[str, set[str]] = {} + for raw in techniques: + clean = normalise(raw) + if not clean: + continue + out.setdefault(parent(clean), set()) + if is_subtechnique(clean): + out[parent(clean)].add(clean) + return {root: sorted(children) for root, children in sorted(out.items())} diff --git a/db/init.sql b/db/init.sql index e8a11d2..fff9bb4 100644 --- a/db/init.sql +++ b/db/init.sql @@ -166,7 +166,7 @@ CREATE TABLE IF NOT EXISTS events_alert ( source TEXT, `timestamp` TEXT, severity TEXT, - score INT, + score VARCHAR(255), categories TEXT, message TEXT, sent TINYINT(1) NOT NULL DEFAULT 0, @@ -314,7 +314,7 @@ CREATE TABLE IF NOT EXISTS hardware_inventory ( name VARCHAR(255), vendor_id VARCHAR(128), product_id VARCHAR(128), - serial_number VARCHAR(128), + serial_number VARCHAR(255), status VARCHAR(32), `timestamp` TIMESTAMP DEFAULT CURRENT_TIMESTAMP, sent TINYINT(1) DEFAULT 0, @@ -382,9 +382,9 @@ CREATE TABLE IF NOT EXISTS network_connections ( id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, pid INT, process_name VARCHAR(255), - local_addr VARCHAR(64), + local_addr VARCHAR(255), local_port INT, - remote_addr VARCHAR(64), + remote_addr VARCHAR(255), remote_port INT, state VARCHAR(32), `timestamp` TIMESTAMP DEFAULT CURRENT_TIMESTAMP, diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 61feb49..1937a29 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -18,7 +18,8 @@ "lucide-react": "^0.577.0", "react": "^18.3.1", "react-dom": "^18.3.1", - "react-router-dom": "^7.13.1" + "react-router-dom": "^7.13.1", + "recharts": "^3.10.1" }, "devDependencies": { "@eslint/js": "^9.13.0", @@ -277,6 +278,32 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@reduxjs/toolkit": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.12.0.tgz", + "integrity": "sha512-KiT+RzZbp6mQET+Mg+h2c97+9j1sNflUxQkIHI7Yuzf6Peu+OYpmkn6nbHWmLLWj+1ZODUJFwGZ7gx3L9R9EOw==", + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "@standard-schema/utils": "^0.3.0", + "immer": "^11.0.0", + "redux": "^5.0.1", + "redux-thunk": "^3.1.0", + "reselect": "^5.1.0" + }, + "peerDependencies": { + "react": "^16.9.0 || ^17.0.0 || ^18 || ^19", + "react-redux": "^7.2.1 || ^8.1.3 || ^9.0.0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-redux": { + "optional": true + } + } + }, "node_modules/@rolldown/binding-android-arm-eabi": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.5.tgz", @@ -539,6 +566,81 @@ "dev": true, "license": "MIT" }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@standard-schema/utils": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@standard-schema/utils/-/utils-0.3.0.tgz", + "integrity": "sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g==", + "license": "MIT" + }, + "node_modules/@types/d3-array": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz", + "integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==", + "license": "MIT" + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", + "license": "MIT" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", + "license": "MIT" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "license": "MIT", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz", + "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", + "license": "MIT" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz", + "integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==", + "license": "MIT", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-kVd74ta9eof3eJOvbNd1vGKS/XERRyQbT26Og63hIsvDO84cjD5gEOhsXf26w3FSoNlPVz84DOFcKv/oou+fMw==", + "license": "MIT", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz", + "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", + "license": "MIT" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -557,14 +659,14 @@ "version": "15.7.15", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/@types/react": { "version": "18.3.28", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.28.tgz", "integrity": "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "@types/prop-types": "*", @@ -581,6 +683,12 @@ "@types/react": "^18.0.0" } }, + "node_modules/@types/use-sync-external-store": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.6.tgz", + "integrity": "sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg==", + "license": "MIT" + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.56.1", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.56.1.tgz", @@ -1072,6 +1180,15 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1143,9 +1260,130 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "dev": true, + "devOptional": true, "license": "MIT" }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz", + "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "license": "ISC", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -1163,6 +1401,12 @@ } } }, + "node_modules/decimal.js-light": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz", + "integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==", + "license": "MIT" + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -1248,6 +1492,18 @@ "node": ">= 0.4" } }, + "node_modules/es-toolkit": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.52.0.tgz", + "integrity": "sha512-XTNEJQh1tY1ZJVcf6ayP/2n4ZPyaHlW2FWs7xvw5ddPuhUVjLD3olQVQS7kf58JbAB48iL0uL/jerTrjtV3lDA==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks", + "tests/types", + "tests/browser-compat" + ] + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -1438,6 +1694,12 @@ "node": ">=0.10.0" } }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -1744,6 +2006,16 @@ "node": ">= 4" } }, + "node_modules/immer": { + "version": "11.1.18", + "resolved": "https://registry.npmjs.org/immer/-/immer-11.1.18.tgz", + "integrity": "sha512-EQyQtLiYW029lyoczMl/Hh4Xu7cDecSc58JRYpHyL4tIAu3eqd1yJzQX04d2BZHDkzFFvm6qJEJWOtfDSWAXbQ==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/immer" + } + }, "node_modules/import-fresh": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", @@ -1771,6 +2043,15 @@ "node": ">=0.8.19" } }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -2441,6 +2722,36 @@ "react": "^18.3.1" } }, + "node_modules/react-is": { + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.2.8.tgz", + "integrity": "sha512-s5un28nYxKJw5gvUHyW5PCC28CvBqLu9r3cWgzHT4Vo/5fqqkFcdRYsGcKf50WMPpjjFZS5d76fn3YCo2njKwQ==", + "license": "MIT", + "peer": true + }, + "node_modules/react-redux": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.3.0.tgz", + "integrity": "sha512-KQopgqFo/p/fgmAs5qz6p5RWaNAzq40WAu7fJIXnQpYxFPbJYtsJPWvGeF2rOBaY/kEuV77AVsX8TsQzKm+A/g==", + "license": "MIT", + "dependencies": { + "@types/use-sync-external-store": "^0.0.6", + "use-sync-external-store": "^1.4.0" + }, + "peerDependencies": { + "@types/react": "^18.2.25 || ^19", + "react": "^18.0 || ^19", + "redux": "^5.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "redux": { + "optional": true + } + } + }, "node_modules/react-router": { "version": "7.18.2", "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.2.tgz", @@ -2479,6 +2790,57 @@ "react-dom": ">=18" } }, + "node_modules/recharts": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/recharts/-/recharts-3.10.1.tgz", + "integrity": "sha512-QXFrvt6IVcw7eeZCoyXTwkIJAX3Dv1nyVhMicXJ47GsGDDpcN8z6o644DibE9XjpBTThtsomLKnTV6lc+cVFUA==", + "license": "MIT", + "workspaces": [ + "www" + ], + "dependencies": { + "@reduxjs/toolkit": "^1.9.0 || 2.x.x", + "clsx": "^2.1.1", + "decimal.js-light": "^2.5.1", + "es-toolkit": "^1.39.3", + "eventemitter3": "^5.0.1", + "immer": "^11.1.8", + "react-redux": "8.x.x || 9.x.x", + "reselect": "5.2.0", + "tiny-invariant": "^1.3.3", + "use-sync-external-store": "^1.2.2", + "victory-vendor": "^37.0.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-is": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/redux": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", + "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", + "license": "MIT" + }, + "node_modules/redux-thunk": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-3.1.0.tgz", + "integrity": "sha512-NW2r5T6ksUKXCabzhL9z+h206HQw/NJkcLm1GPImRQ8IzfXwRGqjVhKJGauHirT0DAuyy6hjdnMZaRoAcy0Klw==", + "license": "MIT", + "peerDependencies": { + "redux": "^5.0.0" + } + }, + "node_modules/reselect": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/reselect/-/reselect-5.2.0.tgz", + "integrity": "sha512-AgZ3UOZm3YndfrJ4OYjgrT7bmCm/1iqkjvEfH/oYjzh6PD2qw4QuT3jjnXIrpdt4MTpMXclMT3lXbmRY+XRakw==", + "license": "MIT" + }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -2597,6 +2959,12 @@ "node": ">=8" } }, + "node_modules/tiny-invariant": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", + "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==", + "license": "MIT" + }, "node_modules/tinyglobby": { "version": "0.2.17", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", @@ -2688,6 +3056,37 @@ "punycode": "^2.1.0" } }, + "node_modules/use-sync-external-store": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz", + "integrity": "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==", + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/victory-vendor": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-37.3.6.tgz", + "integrity": "sha512-SbPDPdDBYp+5MJHhBCAyI7wKM3d5ivekigc2Dk2s7pgbZ9wIgIBYGVw4zGHBml/qTFbexrofXW6Gu4noGxrOwQ==", + "license": "MIT AND ISC", + "dependencies": { + "@types/d3-array": "^3.0.3", + "@types/d3-ease": "^3.0.0", + "@types/d3-interpolate": "^3.0.1", + "@types/d3-scale": "^4.0.2", + "@types/d3-shape": "^3.1.0", + "@types/d3-time": "^3.0.0", + "@types/d3-timer": "^3.0.0", + "d3-array": "^3.1.6", + "d3-ease": "^3.0.1", + "d3-interpolate": "^3.0.1", + "d3-scale": "^4.0.2", + "d3-shape": "^3.1.0", + "d3-time": "^3.0.0", + "d3-timer": "^3.0.1" + } + }, "node_modules/vite": { "version": "8.2.2", "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index 8f1a206..3fa9d6e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,7 +21,8 @@ "lucide-react": "^0.577.0", "react": "^18.3.1", "react-dom": "^18.3.1", - "react-router-dom": "^7.13.1" + "react-router-dom": "^7.13.1", + "recharts": "^3.10.1" }, "devDependencies": { "@eslint/js": "^9.13.0", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index e5ef7d7..1ecbdba 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -8,6 +8,8 @@ import AdminUsers from './pages/AdminUsers'; import AIAnalysis from './pages/AIAnalysis'; import ThreatIntel from './pages/ThreatIntel'; import AttackCoverage from './pages/AttackCoverage'; +import AttackChain from './pages/AttackChain'; +import TelemetryHealth from './pages/TelemetryHealth'; import Login from './pages/Login'; import Playbooks from './pages/Playbooks'; import Databases from './pages/Databases'; @@ -74,6 +76,24 @@ const App: React.FC = () => { } /> + + {/* Per host, so it is reached from the host rather than from the nav: + "which tactics ran on this machine, in what order" is a question + you ask about one machine, not about the estate. */} + + + + } /> + + {/* Also per host: "is this machine's telemetry arriving" is asked + about one machine, and answered by comparing its numbers with + this server's. */} + + + + } /> diff --git a/frontend/src/components/ui/charts.tsx b/frontend/src/components/ui/charts.tsx new file mode 100644 index 0000000..23fccb6 --- /dev/null +++ b/frontend/src/components/ui/charts.tsx @@ -0,0 +1,156 @@ +/** + * Charts that look like the rest of the console. + * + * Recharts styles itself with props, not CSS, so left alone it draws a white + * grid, a blue line and a light tooltip on a black page. These wrappers read + * the same tokens everything else does, which is the only way the theme + * survives contact with a charting library. + * + * Three rules the wrappers enforce rather than leaving to each call site: + * + * Series colours come from `--chart-*` and never from `--sev-*`. A bar being + * red should not imply danger unless danger is what the bar measures. + * + * Axes and grid are quiet. On a data page the reader is looking at the shape; + * a grid that competes with the series is a grid that has to be looked past. + * + * A chart with no data says so. An empty chart and a chart that failed to + * load are the same picture, and on a security console the difference is + * whether you are safe or blind. + */ +import React from 'react'; +import { + Bar, BarChart, CartesianGrid, Cell, ComposedChart, Legend, Line, + Pie, PieChart, ResponsiveContainer, Tooltip, XAxis, YAxis, +} from 'recharts'; +import { EmptyState } from './index'; + +/** Read a CSS custom property, so the palette has one home. */ +const token = (name: string, fallback: string) => { + if (typeof window === 'undefined') return fallback; + const value = getComputedStyle(document.documentElement) + .getPropertyValue(name).trim(); + return value || fallback; +}; + +export const SERIES = [ + 'var(--chart-1)', 'var(--chart-2)', 'var(--chart-3)', + 'var(--chart-4)', 'var(--chart-5)', 'var(--chart-6)', +]; + +const axis = { + stroke: 'var(--chart-axis)', + fontSize: 11, + tickLine: false, +} as const; + +const tooltipStyle = { + contentStyle: { + background: 'var(--surface-2)', + border: '1px solid var(--border-strong)', + borderRadius: 'var(--radius-md)', + fontSize: '0.8125rem', + color: 'var(--text-primary)', + }, + labelStyle: { color: 'var(--text-secondary)' }, + cursor: { fill: 'rgba(255,255,255,0.03)' }, +} as const; + +function Frame({ + height = 220, empty, children, +}: { height?: number; empty: boolean; children: React.ReactElement }) { + if (empty) { + return ( + + ); + } + return ( +
+ + {children} + +
+ ); +} + +/** A count per category. The workhorse. */ +export function CategoryBars({ + data, height, colorFor, +}: { + data: { name: string; value: number }[]; + height?: number; + /** Per-bar colour, for the charts where the category *is* a severity. */ + colorFor?: (row: { name: string; value: number }, index: number) => string; +}) { + return ( + d.value > 0)}> + + + + + + + {data.map((row, i) => ( + + ))} + + + + ); +} + +/** Two counts a day, on two axes, because their shapes mean different things. + * + * A spike in detections with a flat distinct line is almost always one noisy + * rule rather than an incident, and that is only visible when both are drawn + * together. */ +export function TrendChart({ + data, height, +}: { + data: { date: string; detections: number; distinct: number }[]; + height?: number; +}) { + const short = (d: string) => d.slice(5); // MM-DD + return ( + d.detections > 0)}> + + + + + + + + + + + + ); +} + +/** A share of a whole. Used sparingly: a donut answers "what proportion", + * and almost every other question on this console is better as bars. */ +export function ShareDonut({ + data, height, +}: { data: { name: string; value: number }[]; height?: number }) { + return ( + d.value > 0)}> + + + {data.map((row, i) => ( + + ))} + + + + + + ); +} diff --git a/frontend/src/components/ui/index.tsx b/frontend/src/components/ui/index.tsx new file mode 100644 index 0000000..d2a8a66 --- /dev/null +++ b/frontend/src/components/ui/index.tsx @@ -0,0 +1,322 @@ +/** + * The shared shell every page is built from. + * + * There were 1291 inline style objects across 24 pages and `StatCard` was + * written three separate times. That is why the console looked like three + * different products: not because any page was written carelessly, but + * because there was no shared vocabulary to be careless *about*. Spacing, + * radius and colour were decided again on every page, and diverged the way + * independently made decisions always do. + * + * These primitives are deliberately small and unclever. They read tokens from + * index.css and expose almost no styling props, because a component that + * takes `padding` is a component that will be given a different padding on + * every page and we are back where we started. + * + * Responsiveness lives here rather than in each page. A page that lays itself + * out cannot be made responsive twenty times. + */ +import React from 'react'; +import { AlertTriangle, Inbox, Loader2 } from 'lucide-react'; + +/* ── Page shell ─────────────────────────────────────────────────────────── */ + +export function PageHeader({ + title, subtitle, icon, actions, +}: { + title: React.ReactNode; + subtitle?: React.ReactNode; + icon?: React.ReactNode; + actions?: React.ReactNode; +}) { + return ( +
+
+

+ {icon} + {title} +

+ {subtitle && ( +

+ {subtitle} +

+ )} +
+ {actions && ( +
+ {actions} +
+ )} +
+ ); +} + +export function Card({ + children, title, actions, interactive, onClick, style, +}: { + children: React.ReactNode; + title?: React.ReactNode; + actions?: React.ReactNode; + interactive?: boolean; + onClick?: () => void; + style?: React.CSSProperties; +}) { + return ( +
+ {(title || actions) && ( +
+ {title && ( +

+ {title} +

+ )} + {actions} +
+ )} + {children} +
+ ); +} + +/** One number and what it means. + * + * `sub` is not optional decoration. A figure on a security console without a + * sentence saying what it counts is how "23" comes to mean three different + * things on three pages. */ +export function StatCard({ + label, value, sub, color, icon, +}: { + label: string; + value: React.ReactNode; + sub?: string; + color?: string; + icon?: React.ReactNode; +}) { + return ( +
+
+ {icon} + {label} +
+
+ {value} +
+ {sub && ( +
+ {sub} +
+ )} +
+ ); +} + +/* ── States a page is in before it has data ─────────────────────────────── */ + +/** + * The three of these exist together on purpose. + * + * "Loading", "nothing here" and "this failed" were routinely collapsed into + * one blank area, and on a security console that is the worst possible + * ambiguity: an empty alert list and an alert list that failed to load look + * identical, and only one of them means you are safe. + */ +export function LoadingState({ label = 'Loading…' }: { label?: string }) { + return ( +
+ + {label} +
+ ); +} + +export function EmptyState({ + title, detail, icon, +}: { title: string; detail?: string; icon?: React.ReactNode }) { + return ( +
+ {icon ?? } +
+
{title}
+ {detail && ( +
+ {detail} +
+ )} +
+
+ ); +} + +/** Never silently. A page that failed to load must say so and say why. */ +export function ErrorState({ title, detail }: { title: string; detail?: string }) { + return ( +
+ +
+
{title}
+ {detail && ( +
+ {detail} +
+ )} +
+
+ ); +} + +const stateBox: React.CSSProperties = { + display: 'flex', alignItems: 'center', gap: 'var(--space-3)', + padding: 'var(--space-5)', + border: '1px dashed var(--border-color)', + borderRadius: 'var(--radius-md)', + color: 'var(--text-secondary)', + fontSize: 'var(--text-sm)', +}; + +/* ── Small pieces ───────────────────────────────────────────────────────── */ + +export type Tone = 'critical' | 'high' | 'medium' | 'low' | 'info' | 'ok' | 'neutral'; + +const TONE_COLOR: Record = { + critical: 'var(--sev-critical)', + high: 'var(--sev-high)', + medium: 'var(--sev-medium)', + low: 'var(--sev-low)', + info: 'var(--sev-info)', + ok: 'var(--accent-success)', + neutral: 'var(--accent-neutral)', +}; + +export function toneColor(tone: Tone): string { + return TONE_COLOR[tone] ?? TONE_COLOR.neutral; +} + +/** Severity as a label, so no page has to decide which red is critical. */ +export function Badge({ + children, tone = 'neutral', +}: { children: React.ReactNode; tone?: Tone }) { + const color = toneColor(tone); + return ( + + {children} + + ); +} + +/** A table that scrolls sideways rather than breaking the page. + * + * Wide content is the one thing that makes a console unusable on a laptop: + * the body scrolls horizontally, the sidebar drifts, and every column is a + * pixel too narrow. The overflow belongs to the table, not to the page. */ +export function DataTable({ + columns, children, +}: { columns: React.ReactNode[]; children: React.ReactNode }) { + return ( +
+ + + + {columns.map((c, i) => ( + + ))} + + + {children} +
+ {c} +
+
+ ); +} + +export function Row({ children }: { children: React.ReactNode }) { + return {children}; +} + +export function Cell({ + children, mono, align, +}: { children: React.ReactNode; mono?: boolean; align?: 'left' | 'right' }) { + return ( + + {children} + + ); +} diff --git a/frontend/src/index.css b/frontend/src/index.css index d286646..87d3342 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -12,24 +12,101 @@ @import '@fontsource/source-code-pro/600.css'; @import '@fontsource/source-code-pro/700.css'; +/* ── Design tokens ──────────────────────────────────────────────────────── + * + * Black carries the page; white draws the structure. Colour is reserved for + * meaning - severity, state, a series on a chart - and never used for + * decoration, because on a console where red means "something is wrong" a + * decorative red costs an operator a second every time they scan the page. + * + * Everything here exists because 1291 inline style objects across 24 pages + * did not: each page invented its own spacing, radius and colour, so the UI + * could not be consistent no matter how carefully any single page was + * written. These are the shared vocabulary. `components/ui` is the shared + * grammar. + */ :root { + /* Surfaces. Four steps, dark to light, so depth is expressed by value + rather than by shadow - shadows on a black ground are invisible and were + doing nothing but costing paint time. */ --bg-color: #000000; + --surface-1: #080808; + --surface-2: #101010; + --surface-3: #171717; --sidebar-bg: #050505; - --card-bg: rgba(10, 10, 10, 0.8); - --border-color: rgba(255, 255, 255, 0.08); - --border-neon: rgba(255, 255, 255, 0.2); + --card-bg: var(--surface-1); + + /* Structure. White at low alpha: the frame, not the fill. */ + --border-color: rgba(255, 255, 255, 0.10); + --border-strong: rgba(255, 255, 255, 0.18); + --border-neon: var(--border-strong); /* old name, kept so pages still build */ + + /* Text. Three steps is enough; a fourth always gets misused as a second + body colour and the hierarchy stops reading. */ --text-primary: #FAFAFA; - --text-secondary: #A3A3A3; - --accent-color: #EF4444; /* Vibrant Red */ - --accent-secondary: #3B82F6; /* Vibrant Blue */ - --accent-success: #10B981; /* Vibrant Green */ - --accent-warning: #F59E0B; /* Vibrant Amber */ - --shadow-sm: 0 1px 2px 0 rgba(0, 0, 0, 0.05); - --shadow-md: 0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px rgba(0, 0, 0, 0.06); - --shadow-lg: 0 10px 15px -3px rgba(0, 0, 0, 0.1), 0 4px 6px -2px rgba(0, 0, 0, 0.05); - --shadow-glow: 0 0 15px rgba(59, 130, 246, 0.1); - --radius-md: 12px; - --radius-lg: 16px; + --text-secondary: #A1A1AA; + --text-muted: #71717A; + + /* Semantic colour. Desaturated from the previous set: these sit on pure + black, where a saturated hue vibrates and reads as an alarm even when it + is a label. */ + --accent-color: #DC5B5B; /* danger */ + --accent-secondary: #5B8DEF; /* info */ + --accent-success: #4FA97C; /* ok */ + --accent-warning: #D9A441; /* warning */ + --accent-neutral: #8B8B93; /* no state */ + + /* Severity, named for what it means rather than for its hue, so a page + never has to decide which red is the critical one. */ + --sev-critical: #DC5B5B; + --sev-high: #D97A45; + --sev-medium: #D9A441; + --sev-low: #5B8DEF; + --sev-info: #8B8B93; + + /* Chart series. Ordered for distinguishability at small size, and + deliberately not the semantic colours above - a bar being red should not + imply danger unless danger is what the bar measures. */ + --chart-1: #7AA2C4; + --chart-2: #6FA88B; + --chart-3: #C4A46F; + --chart-4: #9B87C4; + --chart-5: #6FA5A8; + --chart-6: #C48A7A; + --chart-grid: rgba(255, 255, 255, 0.06); + --chart-axis: #71717A; + + /* Spacing. A scale, so vertical rhythm is a choice made once instead of + 28px here and 24px three lines down. */ + --space-1: 4px; + --space-2: 8px; + --space-3: 12px; + --space-4: 16px; + --space-5: 24px; + --space-6: 32px; + --space-7: 48px; + + /* Type. */ + --text-xs: 0.75rem; + --text-sm: 0.8125rem; + --text-base: 0.875rem; + --text-lg: 1rem; + --text-xl: 1.25rem; + --text-2xl: 1.5rem; + --text-3xl: 1.875rem; + + /* Radius. One value, small. The old set had 12px cards next to 4px inputs, + which is most of why the console read as a toy rather than a tool. */ + --radius-sm: 3px; + --radius-md: 4px; + --radius-lg: 6px; + + /* Kept because pages reference them. Shadows do nothing on black; these + resolve to nothing rather than being hunted down across 24 files. */ + --shadow-sm: none; + --shadow-md: none; + --shadow-lg: none; + --shadow-glow: none; } * { @@ -118,6 +195,14 @@ body { to { opacity: 1; } } +@keyframes spin { + to { transform: rotate(360deg); } +} + +.spin { + animation: spin 1s linear infinite; +} + h1, h2, h3, h4, h5 { font-family: 'Inter', sans-serif; font-weight: 500; @@ -190,21 +275,31 @@ button:active { border-color: var(--border-neon); } +/* A surface, not an object. + * + * This used to lift two pixels and glow on hover, over a 12px backdrop blur. + * On a page that is mostly cards, that means the whole console moves when the + * pointer crosses it - which reads as a toy, and on a dense table view it is + * genuinely distracting. A card is where data sits; it is not a button. + * + * The blur went with it. It was repainting every card on every scroll to + * produce an effect nobody could see against a black ground. */ .card { background: var(--card-bg); - backdrop-filter: blur(12px); - -webkit-backdrop-filter: blur(12px); border: 1px solid var(--border-color); border-radius: var(--radius-md); - padding: 24px; - box-shadow: var(--shadow-md); - transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1); + padding: var(--space-5); } -.card:hover { - border-color: var(--border-neon); - transform: translateY(-2px); - box-shadow: var(--shadow-lg), var(--shadow-glow); +/* Only where the card is genuinely a control. Pages opt in. */ +.card--interactive { + cursor: pointer; + transition: border-color 0.12s ease, background-color 0.12s ease; +} + +.card--interactive:hover { + border-color: var(--border-strong); + background: var(--surface-2); } input, select, textarea { diff --git a/frontend/src/pages/AgentDetail.tsx b/frontend/src/pages/AgentDetail.tsx index 37ce8f1..3b04c71 100644 --- a/frontend/src/pages/AgentDetail.tsx +++ b/frontend/src/pages/AgentDetail.tsx @@ -20,6 +20,7 @@ import { RotateCcw, Pencil, FileDown, + GitBranch, Bomb, MonitorPlay, TerminalSquare as TerminalIcon, @@ -448,6 +449,21 @@ const AgentDetail: React.FC = () => {
+ {/* Reached from the host, not from the nav: "which tactics ran + here, in what order" is a question about one machine. */} + + Attack chain + + {/* The first place to look when a tab on this page is empty: + it says whether that means "nothing to report" or "the data + is being collected and lost". */} + + Telemetry + + ))} + + } + /> + + {loading && } + {!loading && error && ( + + )} + {!loading && !error && seen.size === 0 && ( + + )} + + {!loading && !error && seen.size > 0 && ( +
+ {order.map((tactic) => { + const stage = seen.get(tactic); + return ( + +
+
+ + {String(order.indexOf(tactic) + 1).padStart(2, '0')} + + + {readable(tactic)} + + {stage + ? {stage.events} events + : + not seen + } +
+ {stage && ( + + {when(stage.first_seen)} → {when(stage.last_seen)} + + )} +
+ + {stage && ( +
+ {stage.techniques.map((t) => ( + + {t} + + ))} +
+ )} + + ); + })} +
+ )} +
+ ); +} diff --git a/frontend/src/pages/AttackCoverage.tsx b/frontend/src/pages/AttackCoverage.tsx index cd8a888..95974b7 100644 --- a/frontend/src/pages/AttackCoverage.tsx +++ b/frontend/src/pages/AttackCoverage.tsx @@ -1,5 +1,10 @@ import React, { useCallback, useEffect, useState } from 'react'; -import { Grid3x3, ShieldAlert, ShieldCheck, EyeOff, AlertTriangle } from 'lucide-react'; +import { Grid3x3, ShieldAlert, ShieldCheck, EyeOff, AlertTriangle, GitBranch } from 'lucide-react'; +// One StatCard, not three. It was written independently on this page, on +// Dashboard and on ThreatIntel, and the three had drifted to different +// paddings, weights and label sizes - which is most of why the console read +// as three products rather than one. +import { StatCard } from '../components/ui'; import { agentService } from '../services/api'; /** @@ -33,6 +38,18 @@ type Coverage = { observed_count: number; quiet: string[]; uncovered_but_seen: string[]; + /** Seen, with rules for *other* sub-techniques of the same parent and not + * this one. Its own state because it is the one most easily mistaken for + * coverage: the parent cell is green while this particular action would go + * unnoticed. Detecting T1003.001 (LSASS memory) says nothing about + * T1003.003 (the AD database) - different action, different telemetry. */ + covered_only_by_a_sibling: string[]; + /** Seen at sub-technique granularity, with a rule claiming the whole + * parent. Genuinely covered. */ + covered_by_the_parent: string[]; + /** Parent technique -> the sub-techniques covered under it, so a rolled-up + * cell can say "2 sub-techniques" rather than implying the whole of it. */ + subtechniques: Record; }; /** The tactics, in the order ATT&CK orders them: earliest to latest in a @@ -70,20 +87,6 @@ const tacticOf = (technique: string): string => const label = (tactic: string): string => tactic.split('-').map(w => w[0].toUpperCase() + w.slice(1)).join(' '); -const StatCard: React.FC<{ - label: string; value: React.ReactNode; sub: string; - color: string; icon: React.ReactNode; -}> = ({ label, value, sub, color, icon }) => ( -
-
- {icon} - {label} -
-
{value}
-
{sub}
-
-); - const AttackCoverage: React.FC = () => { const [data, setData] = useState(null); const [loading, setLoading] = useState(true); @@ -172,8 +175,32 @@ const AttackCoverage: React.FC = () => { sub="surfaced by the AI or the regex list, with no rule behind it" color="#fbbf24" icon={} /> + } + /> + {!loading && !!data?.covered_only_by_a_sibling?.length && ( +
+ {data.covered_only_by_a_sibling.join(', ')} fired here, + and the rules you have cover a different sub-technique of the + same parent. The grid below is drawn at parent granularity, so those + cells read as covered — they are not. Detecting one sub-technique says + nothing about its siblings: they are different actions with different + telemetry, and the rule for one will never fire on the other. +
+ Usually cheaper to widen an existing rule than to write a new one. +
+
+ )} + {!loading && data && data.covered_count === 0 && (
= { + 'No channel': 'var(--sev-critical)', + 'Unsupported agent': 'var(--sev-high)', + 'Version unknown': 'var(--sev-medium)', + 'Behind': 'var(--sev-low)', +}; + +const CONNECTION_TONE: Record = { + Connected: 'var(--accent-success)', + Disconnected: 'var(--sev-critical)', +}; +import { + ShieldAlert, Monitor, Activity, Cpu, @@ -19,6 +40,38 @@ const Dashboard: React.FC = () => { const [dbStatus, setDbStatus] = useState(null); const [summary, setSummary] = useState(null); const [recentAlerts, setRecentAlerts] = useState([]); + const [trend, setTrend] = useState<{ date: string; detections: number; distinct: number }[]>([]); + + /* Derived on the page rather than asked of the server: the agents list + already carries every field these count, and a second endpoint would be + a second place for the definition of "needs attention" to live. */ + const attentionBuckets = useMemo(() => { + const count = (fn: (a: any) => boolean) => agents.filter(fn).length; + return [ + { name: 'No channel', value: count((a) => a.channel_connected === false) }, + { name: 'Unsupported agent', value: count((a) => a.version_state === 'unsupported') }, + { name: 'Version unknown', value: count((a) => a.version_state === 'unknown') }, + { name: 'Behind', value: count((a) => a.version_state === 'behind') }, + ]; + }, [agents]); + + const connectionBuckets = useMemo(() => ([ + { name: 'Connected', value: agents.filter((a: any) => a.channel_connected).length }, + { name: 'Disconnected', value: agents.filter((a: any) => !a.channel_connected).length }, + ]), [agents]); + + const versionSpread = useMemo(() => { + const counts = new Map(); + for (const a of agents as any[]) { + // Null is a real answer, not a gap: it means the agent sent no version + // at all, which is a different fact from being old. + const key = a.agent_version || 'Not reported'; + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return [...counts.entries()] + .map(([name, value]) => ({ name, value })) + .sort((a, b) => b.value - a.value); + }, [agents]); const [loading, setLoading] = useState(true); useEffect(() => { @@ -33,7 +86,7 @@ const Dashboard: React.FC = () => { // Four fan-out requests collapsed into one aggregate. The page was // downloading up to 100 decrypted alerts per agent, and every AI // insight with its full raw log attached, only to count them. - const [agentList, summary, serverRes, gStats, dbRes, aiRes, recent] = await Promise.all([ + const [agentList, summary, serverRes, gStats, dbRes, aiRes, recent, trendRes] = await Promise.all([ agentService.getAgents(), agentService.getDashboardSummary(), agentService.getServerResources(), @@ -43,7 +96,11 @@ const Dashboard: React.FC = () => { agentService.getCustom('/api/ai-insights/all?limit=3&per_agent=3').catch(() => null), // The feed below shows six rows; it used to pull a hundred per agent. agentService.getCustom('/all_alerts?per_agent=3').catch(() => []), + // Fleet-wide detections per day. Its own request because it walks + // every agent database and must not hold up the tiles above it. + agentService.getCustom('/threat-trend?days=14').catch(() => null), ]); + setTrend(trendRes?.series || []); setAgents(agentList || []); setSummary(summary?.status === 'success' ? summary : null); setResources(serverRes); @@ -119,26 +176,26 @@ const Dashboard: React.FC = () => { {/* Main Stats Row */}
- } label="Connected Agents" value={agents.length.toString()} link="/agents" /> - } label="Critical Alerts" value={criticalCount.toString()} warning={criticalCount > 0} link="/all-alerts" /> - } label="Assets Discovered" value={((globalStats?.total_hardware || 0) + (globalStats?.total_software || 0)).toString()} link="/assets" /> - } label="FIM Violations" value={(globalStats?.total_fim_events || 0).toString()} @@ -147,6 +204,46 @@ const Dashboard: React.FC = () => { />
+ {/* Fleet health, as shapes rather than as numbers. + Four counts an operator scans rather than reads: which hosts need + attention, whether commands can actually reach them, what is + installed out there, and whether today is unusual. */} +
+ +

+ Not an ideal operational state. Each bar is a different problem + with a different fix. +

+ ATTENTION_TONE[row.name]} /> + + + +

+ Whether a command would reach the host now. Telemetry travels on a + separate connection, so an agent can be reporting and still be + uncommandable. +

+ CONNECTION_TONE[row.name]} /> + + + +

+ What is actually installed across the fleet. Hosts that report no + version predate version reporting entirely. +

+ + +
+ + +

+ Detections a day against how many different techniques were + behind them. A spike in the bars with a flat line is almost always + one noisy rule rather than an incident — which is why both are drawn. +

+ + +
{/* Recent Alerts Feed */}
@@ -412,26 +509,22 @@ const ResourceCard: React.FC<{ label: string, value: number, icon: React.ReactNo
); -const StatCard: React.FC<{ - icon: React.ReactNode, - label: string, - value: string, - warning?: boolean, - link?: string -}> = ({ icon, label, value, warning, link }) => { - const content = ( -
-
-
- {icon} -
- {label} -
-
{value}
-
+/** A dashboard tile that navigates. The shared `StatCard` deliberately knows + * nothing about routing - a primitive that takes a `link` is a primitive that + * will be asked to take an `onClick`, then a `target`, and stop being one. */ +const LinkedStat = ({ icon, label, value, warning, link }: { + icon: React.ReactNode; label: string; value: string; + warning?: boolean; link?: string; +}) => { + const tile = ( + ); - - return link ? {content} : content; + return link ? {tile} : tile; }; export default Dashboard; diff --git a/frontend/src/pages/TelemetryHealth.tsx b/frontend/src/pages/TelemetryHealth.tsx new file mode 100644 index 0000000..994124a --- /dev/null +++ b/frontend/src/pages/TelemetryHealth.tsx @@ -0,0 +1,181 @@ +/** + * Does this host's telemetry actually arrive? Per table, end to end. + * + * The page exists because five different failures all looked the same from + * the console: an empty table. A collector that never ran, a batch the server + * discarded, a send that has been failing for hours, data collected two + * seconds ago and not yet shipped, and a host that genuinely has nothing to + * report are five different problems needing five different people to do five + * different things — and every one of them rendered as a blank list. + * + * The numbers on either side of each row are the whole point. The agent says + * what it holds and believes it shipped; the server says what it holds. Where + * those disagree, the chain is broken and the row says where. + * + * Broken rows sort first. A page that needs scrolling to find the problem is a + * page nobody opens twice. + */ +import { useCallback, useEffect, useState } from 'react'; +import { useParams, Link } from 'react-router-dom'; +import { Activity, ChevronLeft, RefreshCw } from 'lucide-react'; +import api from '../services/api'; +import { + PageHeader, Card, Badge, DataTable, Row, Cell, + EmptyState, ErrorState, LoadingState, StatCard, Tone, +} from '../components/ui'; + +type TableHealth = { + table: string; + state: string; + detail: string; + agent_held: number | null; + agent_unsent: number | null; + agent_shipped: number | null; + agent_last_error: string | null; + server_rows: number | null; +}; + +/** Severity of each state, and the order they sort in. `lost in transit` is + * first because it is the one that is actively losing data right now. */ +const STATE_TONE: Record = { + 'lost in transit': 'critical', + 'send failing': 'critical', + 'not collected': 'high', + 'no table': 'medium', + queued: 'low', + empty: 'neutral', + flowing: 'ok', +}; + +const STATE_RANK: Record = { + 'lost in transit': 0, + 'send failing': 1, + 'not collected': 2, + 'no table': 3, + queued: 4, + empty: 5, + flowing: 6, +}; + +const rank = (s: string) => STATE_RANK[s] ?? 99; +const tone = (s: string): Tone => STATE_TONE[s] ?? 'neutral'; +const num = (v: number | null) => (v === null || v === undefined ? '—' : v); + +export default function TelemetryHealth() { + const { agentName = '' } = useParams(); + const [rows, setRows] = useState([]); + const [broken, setBroken] = useState(0); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(''); + + const load = useCallback(async () => { + setLoading(true); + setError(''); + try { + const res = await api.get( + `/telemetry-health/${encodeURIComponent(agentName)}`); + setRows(res.data?.tables ?? []); + setBroken(res.data?.broken_count ?? 0); + } catch (e: any) { + // Said out loud, and distinguished from "everything is fine". An + // unreachable agent and a healthy one must never render alike — that + // ambiguity is the reason this page exists. + setError(e?.response?.data?.message || e?.message || 'Request failed'); + } finally { + setLoading(false); + } + }, [agentName]); + + useEffect(() => { load(); }, [load]); + + const sorted = [...rows].sort( + (a, b) => rank(a.state) - rank(b.state) || a.table.localeCompare(b.table)); + const flowing = rows.filter((r) => r.state === 'flowing').length; + + return ( +
+ } + title={`Telemetry health — ${agentName}`} + subtitle="What the agent holds and believes it shipped, beside what this + server actually holds. Where the two disagree, the chain is + broken — and every one of those breaks used to look like an + ordinary empty table." + actions={ + <> + + Back to host + + + + } + /> + + {loading && } + {!loading && error && ( + + )} + + {!loading && !error && ( + <> +
+ + + +
+ + {rows.length === 0 ? ( + + ) : ( + + + {sorted.map((r) => ( + + {r.table} + {r.state} + {num(r.agent_held)} + {num(r.agent_unsent)} + {num(r.agent_shipped)} + {num(r.server_rows)} + + {r.detail} + {r.agent_last_error && ( +
+ {r.agent_last_error} +
+ )} + + + ))} + + + )} + + )} +
+ ); +} diff --git a/frontend/src/pages/ThreatIntel.tsx b/frontend/src/pages/ThreatIntel.tsx index 0f51b88..48f451f 100644 --- a/frontend/src/pages/ThreatIntel.tsx +++ b/frontend/src/pages/ThreatIntel.tsx @@ -1,4 +1,5 @@ import React, { useEffect, useState, useCallback } from 'react'; +import { StatCard } from '../components/ui'; import { Radar, RefreshCw, Search, AlertTriangle, Globe, FileDigit, Link2, Server } from 'lucide-react'; import { agentService, adminService, authService } from '../services/api'; @@ -159,7 +160,7 @@ const ThreatIntel: React.FC = () => { label="Last updated" value={relativeAge(stats?.newest)} sub={stats?.newest ? new Date(stats.newest).toLocaleString() : 'no data yet'} - warn={!stats?.newest} + color={!stats?.newest ? 'var(--accent-warning)' : undefined} /> { ); }; -const StatCard: React.FC<{ label: string; value: React.ReactNode; sub?: string; warn?: boolean }> = ({ label, value, sub, warn }) => ( -
-
- {label} -
-
- {value} -
- {sub &&
{sub}
} -
-); - export default ThreatIntel; diff --git a/server.py b/server.py index 553ac23..ec24f07 100644 --- a/server.py +++ b/server.py @@ -27,13 +27,58 @@ DB_PASSWORD = os.getenv('DB_PASSWORD', 'my-secret-pw') DB_PORT = int(os.getenv('DB_PORT', '3306')) +#: Tables that hold the *current* state of a host rather than a history of +#: events. Each batch replaces the contents: the agent re-sends the whole +#: picture, and the previous picture is no longer true. +SNAPSHOT_TABLES = { + "resource_usage", "disk_usage", "critical_files", + "network_connections", "hardware_inventory", "docker_containers", +} + +#: Tables where the same event must never be stored twice. +#: +#: Disjoint from SNAPSHOT_TABLES, and that is not a style preference - the two +#: mechanisms cancel each other out exactly. A snapshot table is emptied +#: before the batch is written; a deduplicated row is skipped when its +#: fingerprint has been seen before. Put a table in both and the sequence is: +#: +#: batch 1 delete nothing, fingerprints are new, 50 rows written +#: batch 2 delete all 50, every fingerprint already known, 0 rows written +#: batch 3+ the same +#: +#: which leaves the table permanently empty after the second batch, while the +#: agent's log reads `network_connections sent (50 rows)` every cycle. The +#: console showed a host with no network connections and no hardware, the +#: agent showed it shipping both, and both were telling the truth. +#: +#: `_assert_ingest_modes_are_exclusive` below makes adding a table to both a +#: startup failure rather than a silently empty tab. DEDUP_TABLES = { - "critical_files", "portscan_result", "packages", + "portscan_result", "packages", "vulnerabilities_report", "siem_events", "events_alert", "soar_actions", - "fim_data", "registry_logs", "network_connections", "process_events", "hardware_inventory", "security_audit", "docker_containers", + "fim_data", "registry_logs", "process_events", "security_audit", "software_inventory", "network_inventory" } + +def _assert_ingest_modes_are_exclusive() -> None: + """A table is a snapshot or it is deduplicated, never both. + + Checked at import so the failure is a server that will not start, rather + than a tab that is empty for reasons nobody can see from either side. + """ + both = SNAPSHOT_TABLES & DEDUP_TABLES + if both: + raise RuntimeError( + f"{sorted(both)} are both snapshot and deduplicated tables. A " + f"snapshot is emptied before each batch and a duplicate row is " + f"skipped, so together they leave the table permanently empty " + f"after the second batch while the agent reports sending rows." + ) + + +_assert_ingest_modes_are_exclusive() + ALLOWED_TABLES = { "critical_files", "portscan_result", "resource_usage", "disk_usage", "packages", "vulnerabilities_report", @@ -209,6 +254,61 @@ def _migrate_automation_status(cursor, db_name: str) -> None: f"will retry on the next start") +#: Columns holding Fernet ciphertext that shipped too narrow to hold it. +#: +#: `network_connections.local_addr` was VARCHAR(64). A token is over a hundred +#: characters whatever the plaintext, so every batch failed on its first row +#: with `1406 Data too long` - and because that table is a snapshot, the +#: DELETE had already run. The tab was empty for days while the agent's log +#: read `network_connections sent (50 rows)` every cycle. +#: +#: `tests/test_encrypted_column_widths.py` compares the two declarations that +#: have to agree, so the next one fails in CI rather than as an empty tab. +_WIDEN_FOR_CIPHERTEXT = ( + ("network_connections", "local_addr", "VARCHAR(255)"), + ("network_connections", "remote_addr", "VARCHAR(255)"), + ("hardware_inventory", "serial_number", "VARCHAR(255)"), + ("events_alert", "score", "VARCHAR(255)"), +) + + +def _migrate_encrypted_widths(cursor, db_name: str) -> None: + """Widen encrypted columns on databases that already exist. + + `CREATE TABLE IF NOT EXISTS` leaves an older table exactly as it was, so + fixing the schema file only helps deployments made after today. + """ + cursor.execute( + "SELECT table_name, column_name, character_maximum_length " + "FROM information_schema.columns WHERE table_schema = %s", + (db_name,), + ) + widths = {(str(t).lower(), str(c).lower()): m for t, c, m in cursor.fetchall()} + + for table, column, declaration in _WIDEN_FOR_CIPHERTEXT: + current = widths.get((table.lower(), column.lower()), None) + # Absent (no such column) or already wide enough: nothing to do. + if current is None or int(current) >= 200: + continue + try: + cursor.execute("SET SESSION lock_wait_timeout = 3") + # Backticks directly, as everywhere else in this file. The names + # come from the constant above, not from anything an agent sends. + cursor.execute( + f"ALTER TABLE `{table}` MODIFY COLUMN `{column}` {declaration} NULL") + print(f"[+] {db_name}: {table}.{column} widened to hold ciphertext", + flush=True) + except Exception as e: + # Deliberately not just `mysql.connector.Error`. The first version + # of this called a helper that does not exist in this module, so it + # raised NameError - which that clause does not catch, on a path + # only reached when there is work to do. The migration died + # silently every time it mattered and never when it did not. + print(f"[!] {db_name}: could not widen {table}.{column} " + f"({type(e).__name__}: {e}); encrypted rows will keep being " + f"rejected", flush=True) + + def _migrate_agent_info(cursor, db_name: str) -> None: """Bring an older agent_info up to what the check-in INSERT writes. @@ -467,6 +567,11 @@ def create_tables_if_not_exist(db_name): except mysql.connector.Error as e: print(f"[!] soar_actions migration skipped: {e}") + try: + _migrate_encrypted_widths(cursor, db_name) + except mysql.connector.Error as e: + print(f"[!] encrypted-width migration skipped: {e}") + try: # Every column update_agent_info's INSERT names is declared here. # @@ -622,7 +727,10 @@ async def insert_data(agent: str, table: str, data: list, public_ip: str = None, print(f"[!] Unknown table '{table}' received. Skipping.") return - if table in {"resource_usage", "disk_usage", "critical_files", "network_connections", "hardware_inventory", "docker_containers"}: + # A snapshot replaces what was there. Named rather than written out + # here, because the same list existing twice is how it came to + # overlap with DEDUP_TABLES in the first place. + if table in SNAPSHOT_TABLES: cursor.execute(f"DELETE FROM `{table}`") for item in data: diff --git a/tests/test_attack_hierarchy.py b/tests/test_attack_hierarchy.py new file mode 100644 index 0000000..6a82570 --- /dev/null +++ b/tests/test_attack_hierarchy.py @@ -0,0 +1,210 @@ +"""Techniques are a hierarchy, and coverage does not roll up freely. + +The coverage endpoint compared techniques as opaque strings: + + "quiet": set(covered) - set(observed) + "uncovered_but_seen": set(observed) - set(covered) + +So a rule tagged `T1003.001` and an event carrying `T1003` never matched each +other. One landed in "covered but never seen", the other in "seen with nothing +covering it", and both were wrong - and those two lists are the only ones an +operator acts on. One sends somebody to test a detection that already works; +the other sends them to write a rule that already exists. + +The fix is not "compare parents". Detecting `T1003.001` (LSASS memory) says +nothing whatsoever about `T1003.003` (the AD database): different action, +different telemetry, and the rule for one will never fire on the other. So +coverage travels *down* from a parent and only partially *up* from a child, +and a heatmap that ignores the asymmetry turns eleven green cells into a +number nobody should trust. +""" +from __future__ import annotations + +import ast +import pathlib + +import pytest + +from core import attack + +ROOT = pathlib.Path(__file__).resolve().parent.parent +APP = ROOT / "app.py" + + +# -------------------------------------------------------------------------- +# Parsing +# -------------------------------------------------------------------------- + +@pytest.mark.parametrize("raw,expected", [ + ("T1059.001", "T1059.001"), + ("t1059.001", "T1059.001"), + (" T1059 ", "T1059"), + ("attack.t1059", ""), # the tag form is stripped before this + ("T105", ""), + ("T10590", ""), + ("T1059.1", ""), + ("", ""), + (None, ""), + ("'; DROP TABLE siem_events; --", ""), +]) +def test_a_technique_is_parsed_or_dropped(raw, expected): + """Total on purpose. These arrive from rule tags, from the AI path and + from a regex list, and a malformed one is a thing to drop rather than to + raise on halfway through building a coverage report.""" + assert attack.normalise(raw) == expected + + +def test_a_parent_is_its_own_parent(): + assert attack.parent("T1059.001") == "T1059" + assert attack.parent("T1059") == "T1059" + assert attack.parent("nonsense") == "" + + +def test_subtechniques_are_recognised(): + assert attack.is_subtechnique("T1003.001") + assert not attack.is_subtechnique("T1003") + + +# -------------------------------------------------------------------------- +# The asymmetry +# -------------------------------------------------------------------------- + +def test_a_rule_for_the_parent_covers_its_children(): + """A rule tagged with a bare technique claims the whole thing.""" + assert attack.coverage_of("T1003.001", {"T1003"}) == "parent" + + +def test_a_rule_for_one_child_does_not_cover_its_sibling(): + """The case this file exists for. Detecting LSASS memory says nothing + about the AD database - and if the two were folded together, the parent + cell would be green while the second action went unnoticed.""" + assert attack.coverage_of("T1003.003", {"T1003.001"}) == "sibling" + + +def test_a_sibling_is_not_reported_as_uncovered_either(): + """`none` and `sibling` need different work: one needs a rule written, + the other usually needs an existing rule widened. Folding them together + loses the cheaper of the two fixes.""" + states = attack.classify({"T1003.001"}, {"T1003.003", "T1486"}) + assert states["sibling"] == ["T1003.003"] + assert states["none"] == ["T1486"] + + +def test_an_exact_match_is_exact(): + assert attack.coverage_of("T1003.001", {"T1003.001"}) == "covered" + + +def test_nothing_related_is_nothing(): + assert attack.coverage_of("T1486", {"T1003.001", "T1059"}) == "none" + + +# -------------------------------------------------------------------------- +# What has never fired +# -------------------------------------------------------------------------- + +def test_an_observation_at_parent_granularity_exercises_the_children(): + """A rule tagged T1003.001 has been exercised by an event carrying + T1003. Calling it unseen sends somebody to test a detection that is + already working.""" + assert attack.unseen({"T1003.001"}, {"T1003"}) == [] + + +def test_a_rule_nothing_has_triggered_is_still_reported(): + assert attack.unseen({"T1486"}, {"T1003"}) == ["T1486"] + + +def test_an_exact_observation_is_not_quiet(): + assert attack.unseen({"T1059.004"}, {"T1059.004"}) == [] + + +# -------------------------------------------------------------------------- +# Tactics as an order +# -------------------------------------------------------------------------- + +def test_the_chain_runs_in_kill_chain_order(): + """"execution then persistence" and "persistence then execution" are + different sentences, and only one of them is a foothold.""" + ordered = attack.order_tactics( + ["impact", "execution", "initial-access", "persistence"]) + assert ordered == ["initial-access", "execution", "persistence", "impact"] + + +def test_an_unknown_tactic_sorts_last_rather_than_vanishing(): + """Something this file has not heard of still happened on the host.""" + ordered = attack.order_tactics(["impact", "wat", "execution"]) + assert ordered[-1] == "wat" + + +def test_the_order_is_the_enterprise_one(): + """Spot-checked at the ends rather than pinned in full, so adding a + tactic does not fail this for the wrong reason.""" + assert attack.TACTIC_ORDER[0] == "reconnaissance" + assert attack.TACTIC_ORDER[-1] == "impact" + assert attack.tactic_position("initial-access") < attack.tactic_position("impact") + + +# -------------------------------------------------------------------------- +# Rolling up for the grid +# -------------------------------------------------------------------------- + +def test_a_rolled_up_cell_remembers_what_it_rolled_up(): + """The grid is drawn at parent granularity because that is the only way + it fits. Without this a cell implies the whole of T1003 when what is + actually covered is one sub-technique of it.""" + rolled = attack.rollup(["T1003.001", "T1003.003", "T1059"]) + assert rolled["T1003"] == ["T1003.001", "T1003.003"] + assert rolled["T1059"] == [] + + +# -------------------------------------------------------------------------- +# The endpoint uses it +# -------------------------------------------------------------------------- + +def _function(name: str) -> str: + tree = ast.parse(APP.read_text(encoding="utf-8")) + return next(ast.unparse(n) for n in ast.walk(tree) + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + and n.name == name) + + +def test_coverage_no_longer_subtracts_sets_of_strings(): + code = _function("get_attack_coverage") + assert "attack.classify" in code + assert "attack.unseen" in code + assert "set(covered) - set(observed)" not in code + assert "set(observed) - set(covered)" not in code + + +def test_the_sibling_state_reaches_the_console(): + """It is the state most easily mistaken for coverage, so it has to be + something the page can show rather than an internal distinction.""" + code = _function("get_attack_coverage") + assert "covered_only_by_a_sibling" in code + + +def test_the_chain_endpoint_returns_the_full_order(): + """So the UI can draw the stages that did *not* happen as gaps. A chain + with holes in it is the readable thing; a list of four tactics is not.""" + code = _function("get_attack_chain") + assert "tactic_order" in code + assert "TACTIC_ORDER" in code + + +def test_the_chain_is_ordered_by_the_chain_not_the_clock(): + code = _function("_observed_chain") + assert "tactic_position" in code + + +def test_a_technique_with_no_tactic_still_appears(): + """Dropping it would make an unmapped rule invisible in exactly the view + meant to show what happened.""" + assert '"other"' in _function("_observed_chain") \ + or "'other'" in _function("_observed_chain") + + +def test_the_tactic_mapping_is_read_from_the_rules(): + """No table here to drift out of date - the same reason the coverage + index reads them rather than keeping a list.""" + code = _function("_sigma_tactics_by_technique") + assert "load_dir" in code + assert "rule.tactics" in code diff --git a/tests/test_collector_isolation.py b/tests/test_collector_isolation.py new file mode 100644 index 0000000..f6bae29 --- /dev/null +++ b/tests/test_collector_isolation.py @@ -0,0 +1,149 @@ +"""An empty table must mean "nothing to report", never "the collector broke". + +The console showed no hardware inventory and no network connections at the +same time. Both collectors were running, both had tables, and nothing anywhere +said anything was wrong. + +`edr_enforcer.main` called five collectors as five bare statements in a row: + + check_fim() + track_network() + monitor_processes() + get_hardware_inventory() + monitor_registry() + +so the first one to raise took the rest of the cycle with it - and because the +caller retries on a timer, it took them again every two minutes, for the life +of the agent. Network is second and hardware is fourth, which is why those two +went dark together. + +Two layers of silence underneath it. The collectors themselves ended in +`except Exception: pass`, and `periodic_wrapped` reported failures behind +`if debug`, which nothing sets. + +The result was a security console showing a host with no hardware, no network +connections and no explanation - which reads as a quiet machine, and is the +one thing it did not mean. +""" +from __future__ import annotations + +import ast +import pathlib + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +EDR = ROOT / "Sentora" / "modules" / "edr_enforcer.py" +MAIN = ROOT / "Sentora" / "main.py" + + +def _function(path: pathlib.Path, name: str) -> ast.FunctionDef: + tree = ast.parse(path.read_text(encoding="utf-8")) + return next(n for n in ast.walk(tree) + if isinstance(n, ast.FunctionDef) and n.name == name) + + +def _code(path: pathlib.Path, name: str) -> str: + """A function's statements, docstring dropped - these explain the bug they + fixed, and matching prose finds the explanation.""" + fn = _function(path, name) + body = fn.body + if (body and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant)): + body = body[1:] + return "\n".join(ast.unparse(n) for n in body) + + +# -------------------------------------------------------------------------- +# One collector failing costs its own data and nothing else +# -------------------------------------------------------------------------- + +def test_the_collectors_are_isolated_from_each_other(): + code = _code(EDR, "main") + assert "try:" in code, ( + "the collectors run as bare statements, so the first to raise " + "silently disables every one after it - permanently, because the " + "caller retries on a timer" + ) + assert "except Exception" in code + + +def test_every_collector_still_runs(): + """Isolating them must not quietly drop one.""" + code = _code(EDR, "main") + for collector in ("check_fim", "track_network", "monitor_processes", + "get_hardware_inventory", "monitor_registry"): + assert collector in code, collector + + +def test_a_failing_collector_names_itself(): + """"something in edr_enforcer failed" sends somebody to read five + functions. The name is the whole diagnosis.""" + code = _code(EDR, "main") + assert "collector failed" in code + assert "print(" in code + + +# -------------------------------------------------------------------------- +# Nothing swallows its own failure +# -------------------------------------------------------------------------- + +@pytest.mark.parametrize("name", [ + "get_hardware_inventory", "monitor_registry", +]) +def test_the_collector_does_not_swallow_everything(name): + """`except Exception: pass` on a collector turns a broken PowerShell call + into an empty inventory tab, and an empty inventory tab into "this host + has no hardware".""" + code = _code(EDR, name) + assert "except Exception:\n pass" not in code, name + assert "pass" not in code.split("except")[-1] if "except" in code else True + + +def test_a_failed_powershell_call_is_an_error_not_an_empty_result(): + """The exit code was checked and then thrown away: `if returncode == 0` + with no else, inside a try that passed. A command that failed and a + command that returned nothing produced the same empty table.""" + code = _code(EDR, "get_hardware_inventory") + assert "raise" in code + assert "returncode" in code + + +# -------------------------------------------------------------------------- +# The runner says when a collector stops working +# -------------------------------------------------------------------------- + +def test_a_failing_collector_is_reported_without_debug(): + """It was behind `if debug`, which nothing in this repository sets. A + collector that raised on its first line failed silently every interval + for the life of the agent.""" + code = _code(MAIN, "periodic_wrapped") + printed = code[code.index("except"):] + assert "print(" in printed + # The print must not be conditional on debug. + lines = [l.strip() for l in printed.splitlines()] + for i, line in enumerate(lines): + if line.startswith("print("): + preceding = " ".join(lines[max(0, i - 2):i]) + assert "if debug" not in preceding, \ + "the failure report is still behind the debug flag" + + +def test_the_report_is_rate_limited(): + """The same argument that makes silence wrong makes a line every two + minutes useless: it would bury the schema errors and auth rejections that + share this log.""" + code = _code(MAIN, "periodic_wrapped") + assert "_FAILURE_REPEAT_EVERY" in code or "%" in code + + +def test_recovery_is_reported_too(): + """Otherwise the log says a collector broke and never says it came back, + and somebody chases a problem that fixed itself an hour ago.""" + code = _code(MAIN, "periodic_wrapped") + assert "recovered" in code + + +def test_the_failure_count_resets_on_success(): + code = _code(MAIN, "periodic_wrapped") + assert "failures = 0" in code diff --git a/tests/test_design_system.py b/tests/test_design_system.py new file mode 100644 index 0000000..d5d5850 --- /dev/null +++ b/tests/test_design_system.py @@ -0,0 +1,241 @@ +"""The console has one vocabulary, and pages use it rather than reinventing it. + +There were 1291 inline style objects across 24 pages, and `StatCard` had been +written three separate times - on Dashboard, ThreatIntel and AttackCoverage - +which had drifted to different paddings, weights and label sizes. + +None of those pages was written carelessly. The problem is that there was no +shared vocabulary to be careless *about*: spacing, radius and colour were +decided again on every page, and independently made decisions diverge. That is +why the console looked like three products. + +These tests do not try to eliminate inline styles - a page will always have +some, and a rule against them would be gamed rather than followed. They pin +the things that actually caused the drift: one definition of each primitive, +one radius scale, and no page inventing a component that already exists. +""" +from __future__ import annotations + +import pathlib +import re + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +FRONTEND = ROOT / "frontend" / "src" +UI = FRONTEND / "components" / "ui" / "index.tsx" +CSS = FRONTEND / "index.css" + +PAGES = sorted((FRONTEND / "pages").glob("*.tsx")) + + +def test_the_primitives_exist(): + source = UI.read_text(encoding="utf-8") + for name in ("PageHeader", "Card", "StatCard", "DataTable", + "EmptyState", "ErrorState", "LoadingState", "Badge"): + assert f"export function {name}" in source, name + + +@pytest.mark.parametrize("page", PAGES, ids=lambda p: p.stem) +def test_no_page_defines_its_own_statcard(page): + """It was defined three times. Each copy looked reasonable; together they + made one number mean three different things visually.""" + source = page.read_text(encoding="utf-8") + assert "const StatCard" not in source, ( + f"{page.name} defines its own StatCard - import it from components/ui" + ) + assert "function StatCard" not in source, page.name + + +def test_the_shared_statcard_is_actually_used(): + """A primitive nothing imports is a primitive that will drift back out.""" + users = [p.name for p in PAGES + if "from '../components/ui'" in p.read_text(encoding="utf-8")] + assert users, "nothing imports the shared components" + + +# -------------------------------------------------------------------------- +# The tokens +# -------------------------------------------------------------------------- + +def _tokens() -> dict: + css = CSS.read_text(encoding="utf-8") + block = css[css.index(":root {"):css.index("\n}", css.index(":root {"))] + return dict(re.findall(r"(--[\w-]+):\s*([^;]+);", block)) + + +@pytest.mark.parametrize("name", [ + "--space-1", "--space-4", "--space-6", + "--text-xs", "--text-base", "--text-2xl", + "--radius-sm", "--radius-md", + "--sev-critical", "--sev-high", "--sev-medium", "--sev-low", + "--chart-1", "--chart-2", "--chart-grid", +]) +def test_the_scale_exists(name): + """Spacing and type as scales, so vertical rhythm is a choice made once + instead of 28px here and 24px three lines down.""" + assert name in _tokens(), f"{name} is not defined" + + +def test_the_radius_scale_is_small(): + """The old set had 12px cards next to 4px inputs, which is most of why the + console read as a toy rather than a tool. One small scale, applied + everywhere.""" + tokens = _tokens() + for name in ("--radius-sm", "--radius-md", "--radius-lg"): + pixels = int(re.sub(r"\D", "", tokens[name])) + assert pixels <= 8, f"{name} is {pixels}px" + + +def test_there_is_a_chart_palette_separate_from_the_semantic_one(): + """A bar being red should not imply danger unless danger is what the bar + measures. Reusing the severity colours for series is how a chart starts + lying about its own data.""" + tokens = _tokens() + series = {tokens[f"--chart-{i}"].strip().lower() for i in range(1, 7)} + severity = {tokens[n].strip().lower() for n in + ("--sev-critical", "--sev-high", "--sev-medium")} + assert not (series & severity), \ + f"chart series reuse severity colours: {sorted(series & severity)}" + + +# -------------------------------------------------------------------------- +# Affordances a data console should not have +# -------------------------------------------------------------------------- + +def _card_rule() -> str: + """The top-level `.card` rule. + + Matched at the start of a line: there is a `.card` override inside a + media query too, and slicing from the first occurrence lands in it and + then spans half the stylesheet - which made this test read `button:active + { transform: ... }` as a card that moves. + """ + css = CSS.read_text(encoding="utf-8") + match = re.search(r"^\.card \{(.*?)^\}", css, re.M | re.S) + assert match, "no top-level .card rule" + return match.group(1) + + +def test_cards_do_not_move_when_you_point_at_them(): + """`.card:hover` lifted two pixels and glowed. On a page that is mostly + cards, the whole console moves when the pointer crosses it - which reads + as a toy, and on a dense table view is genuinely distracting. A card is + where data sits; it is not a button. + + Pages opt into `.card--interactive` where a card really is a control. + """ + css = CSS.read_text(encoding="utf-8") + assert not re.search(r"^\.card:hover", css, re.M), \ + "the card itself reacts to the pointer again" + assert ".card--interactive:hover" in css, \ + "nothing is left for a card that genuinely is a control" + + +def test_cards_do_not_blur_what_is_behind_them(): + """A 12px backdrop blur on every card, repainting on every scroll, to + produce an effect invisible against a black ground.""" + assert "backdrop-filter" not in _card_rule() + + +# -------------------------------------------------------------------------- +# Pages that exist have to be reachable +# -------------------------------------------------------------------------- + +def test_every_page_is_routed(): + """A page nobody can navigate to is a page nobody will find. The chain + view shipped with an endpoint and a component and no way in, which is + indistinguishable from not having built it.""" + app = (FRONTEND / "App.tsx").read_text(encoding="utf-8") + unrouted = [] + for page in PAGES: + if page.stem in ("Login",): + continue + if f"pages/{page.stem}'" not in app: + unrouted.append(page.stem) + assert not unrouted, f"pages with no route: {unrouted}" + + +# -------------------------------------------------------------------------- +# Charts +# -------------------------------------------------------------------------- + +CHARTS = FRONTEND / "components" / "ui" / "charts.tsx" + + +def test_the_chart_library_is_declared(): + """`import 'recharts'` with nothing in package.json is a build that fails + on someone else's machine.""" + package = (ROOT / "frontend" / "package.json").read_text(encoding="utf-8") + assert '"recharts"' in package + + +def test_charts_read_the_theme_rather_than_carrying_their_own(): + """Recharts styles itself with props, not CSS, so left alone it draws a + white grid and a light tooltip on a black page. If the wrappers stop + reading tokens the charts stop matching the console and nothing fails.""" + source = CHARTS.read_text(encoding="utf-8") + assert "var(--chart-1)" in source + assert "var(--chart-grid)" in source + assert "var(--surface-2)" in source + + +def test_series_colours_are_not_the_severity_colours(): + """A bar being red should not imply danger unless danger is what the bar + measures.""" + source = CHARTS.read_text(encoding="utf-8") + series_block = source[source.index("export const SERIES"): + source.index("const axis")] + assert "--sev-" not in series_block + + +def test_an_empty_chart_says_it_is_empty(): + """An empty chart and a chart that failed to load are the same picture, + and on a security console the difference is whether you are safe or + blind.""" + source = CHARTS.read_text(encoding="utf-8") + assert "EmptyState" in source + assert "not a loading state" in source + + +def test_the_trend_draws_both_counts(): + """A thousand detections from one noisy rule and a thousand from forty + techniques are the same bar and completely different days. Only the two + together separate them.""" + source = CHARTS.read_text(encoding="utf-8") + trend = source[source.index("export function TrendChart"): + source.index("export function ShareDonut")] + assert "detections" in trend + assert "distinct" in trend + assert 'yAxisId="right"' in trend, "both series are on one axis" + + +def test_the_trend_endpoint_fills_the_quiet_days(): + """A line drawn only through the days that had events invents a slope + between them, and a two-day gap reads as a gradual decline.""" + app_py = (ROOT / "app.py").read_text(encoding="utf-8") + body = app_py[app_py.index("def _threat_trend"): + app_py.index("@app.route(\"/threat-trend\")")] + assert "timedelta" in body + assert "range(days" in body + + +def test_the_dashboard_says_whether_a_command_would_reach_a_host(): + """Telemetry and control travel on different connections, so an agent can + be reporting and be uncommandable - and the console has been unable to + say which since the HTTP fallback went.""" + app_py = (ROOT / "app.py").read_text(encoding="utf-8") + assert '"channel_connected"' in app_py + + dashboard = (FRONTEND / "pages" / "Dashboard.tsx").read_text(encoding="utf-8") + assert "channel_connected" in dashboard + + +def test_the_attack_chain_is_reachable_from_a_host(): + """It answers a question about one machine, so it is reached from that + machine rather than from the nav.""" + app = (FRONTEND / "App.tsx").read_text(encoding="utf-8") + assert "/agent/:agentName/chain" in app + + detail = (FRONTEND / "pages" / "AgentDetail.tsx").read_text(encoding="utf-8") + assert "/chain" in detail, "no link into the chain view from the host page" diff --git a/tests/test_encrypted_column_widths.py b/tests/test_encrypted_column_widths.py new file mode 100644 index 0000000..5fefb1c --- /dev/null +++ b/tests/test_encrypted_column_widths.py @@ -0,0 +1,160 @@ +"""An encrypted column has to be wide enough for the ciphertext. + +The Network tab was empty for days. The agent's log said + + [+] network_connections sent (50 rows) + +and the ingest log said + + [!] Data insertion error: 1406 (22001): Data too long for column + 'local_addr' at row 1 + +`local_addr` is `VARCHAR(64)` and is one of the fields `enc_db` encrypts. A +Fernet token carries a version byte, a timestamp, a 16-byte IV, the padded +ciphertext and a 32-byte HMAC, base64-encoded - so even a value like +`10.0.0.7` comes out over a hundred characters, plus the `enc::` prefix. +Every batch failed on its first row. + +Worse than a rejected batch: `network_connections` is a snapshot table, so +`DELETE FROM` had already run before the insert failed. Each cycle emptied the +table and then failed to refill it. + +Nothing connected the two logs. The agent reported success because `sendall` +returned, the server reported an error into a log nobody was reading, and the +console showed an empty tab that reads as "this host has no connections". + +This test compares the two declarations that have to agree - which fields are +encrypted, and how wide the columns are - and fails on the mismatch rather +than on the empty tab three layers away. +""" +from __future__ import annotations + +import ast +import pathlib +import re + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +SCHEMA = ROOT / "db" / "init.sql" +APP = ROOT / "app.py" + +#: The shortest Fernet token, base64, plus the `enc::` marker. +#: +#: A token is 1 + 8 + 16 + 16 + 32 = 73 bytes before the plaintext is padded +#: in, which is 100 base64 characters; `enc::` makes 105. Anything narrower +#: cannot hold the encryption of even an empty string. 200 leaves room for a +#: short value without being so generous that a genuinely small column passes +#: by accident. +MIN_ENCRYPTED_WIDTH = 200 + +#: Types with no length to check. +UNBOUNDED = {"TEXT", "TINYTEXT", "MEDIUMTEXT", "LONGTEXT", "BLOB", "JSON"} + + +def _encrypted_fields() -> dict: + tree = ast.parse(APP.read_text(encoding="utf-8")) + for node in tree.body: + if (isinstance(node, ast.Assign) + and getattr(node.targets[0], "id", "") == "ENCRYPTED_FIELDS_MAP"): + return ast.literal_eval(node.value) + raise AssertionError("ENCRYPTED_FIELDS_MAP is not a literal in app.py") + + +def _column_type(table: str, column: str): + """(type, declared length) for one column, or None if it is not there.""" + schema = SCHEMA.read_text(encoding="utf-8") + block = re.search( + rf"CREATE TABLE IF NOT EXISTS {re.escape(table)}\s*\((.*?)\n\)", + schema, re.S) + if not block: + return None + match = re.search( + rf"[`\s]{re.escape(column)}[`\s]+([A-Za-z]+)(?:\((\d+)\))?", + block.group(1)) + if not match: + return None + return match.group(1).upper(), (int(match.group(2)) if match.group(2) else None) + + +CASES = [ + pytest.param(table, column, id=f"{table}.{column}") + for table, columns in _encrypted_fields().items() + for column in columns +] + + +@pytest.mark.parametrize("table,column", CASES) +def test_an_encrypted_column_can_hold_its_ciphertext(table, column): + declared = _column_type(table, column) + if declared is None: + pytest.skip(f"{table}.{column} is not in db/init.sql") + + kind, width = declared + if kind in UNBOUNDED: + return + assert width is not None, f"{table}.{column} is {kind} with no length" + assert width >= MIN_ENCRYPTED_WIDTH, ( + f"{table}.{column} is {kind}({width}) and holds Fernet ciphertext, " + f"which is at least {MIN_ENCRYPTED_WIDTH} characters. Every insert " + f"fails with 'Data too long', and on a snapshot table the DELETE has " + f"already run - so the table is emptied and never refilled, while the " + f"agent reports the rows as sent." + ) + + +def test_the_two_declarations_are_actually_compared(): + """The premise. If either list stops being a literal this file starts + passing vacuously, which is the failure mode of every test that reads + source.""" + assert _encrypted_fields(), "no encrypted fields found" + assert len(CASES) >= 20, f"only {len(CASES)} encrypted columns found" + + +def test_encrypted_reads_decrypt(): + """A route reading an encrypted table has to be given its fields. + + `get_network_inventory` was not, and nowhere else was missing them - so + the Network tab rendered `enc::gAAAAAB...` where the process name should + be. Ciphertext in a column reads as data rather than as a fault: the page + looked populated and was unusable, which is a worse failure than an empty + one because nothing about it says something is wrong. + """ + source = APP.read_text(encoding="utf-8") + tree = ast.parse(source) + encrypted = _encrypted_fields() + + offenders = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + name = getattr(node.func, "id", "") + if name not in ("stream_from_db", "stream_from_db_dec"): + continue + if not node.args or not isinstance(node.args[0], ast.Constant): + continue + table = node.args[0].value + if table not in encrypted: + continue + passes = any(kw.arg == "encrypted_fields" for kw in node.keywords) + if name == "stream_from_db" or not passes: + offenders.append(table) + + assert not offenders, ( + f"{sorted(set(offenders))} hold encrypted columns and are read " + f"without decrypting them; the console will show `enc::...`" + ) + + +def test_an_encrypted_field_names_a_real_column(): + """A field encrypted on the agent and absent from the server's schema is + written nowhere and read as plaintext - the mirror of this bug.""" + missing = [] + for table, columns in _encrypted_fields().items(): + schema = SCHEMA.read_text(encoding="utf-8") + if f"CREATE TABLE IF NOT EXISTS {table}" not in schema: + continue # the table lives elsewhere + for column in columns: + if _column_type(table, column) is None: + missing.append(f"{table}.{column}") + assert not missing, f"encrypted fields with no column: {missing}" diff --git a/tests/test_ingest_modes.py b/tests/test_ingest_modes.py new file mode 100644 index 0000000..06c7f1f --- /dev/null +++ b/tests/test_ingest_modes.py @@ -0,0 +1,131 @@ +"""A table is a snapshot or it is deduplicated. Never both. + +The console showed a host with no network connections and no hardware +inventory. The agent's own log showed it shipping them every cycle: + + [+] network_connections sent (50 rows) + [+] hardware_inventory sent (50 rows) + +Both were telling the truth. `network_connections`, `hardware_inventory`, +`critical_files` and `docker_containers` were in two sets at once, and the two +mechanisms cancel exactly: + + batch 1 DELETE removes nothing, every fingerprint is new, 50 rows land + batch 2 DELETE removes all 50, every fingerprint is known, 0 rows land + batch 3+ the same + +So the table was full once and then permanently empty, while everything +upstream reported success. "It used to be populated" is not a coincidence - +it is the first batch. + +Each mechanism is right on its own. A snapshot holds what is true *now*, so +replacing it is the point and deduplicating it defeats the point: the agent +re-sends the same picture deliberately. A deduplicated table holds a history +of events, so skipping a repeat is the point and emptying it first defeats +that. The bug was not either rule; it was a table subject to both. +""" +from __future__ import annotations + +import ast +import pathlib + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +SERVER = ROOT / "server.py" + + +def _set_literal(name: str) -> set: + tree = ast.parse(SERVER.read_text(encoding="utf-8")) + for node in tree.body: + if isinstance(node, ast.Assign) and getattr(node.targets[0], "id", "") == name: + return set(ast.literal_eval(node.value)) + raise AssertionError(f"{name} is not a module-level set in server.py") + + +def test_the_two_modes_do_not_overlap(): + """The bug, stated as an assertion.""" + both = _set_literal("SNAPSHOT_TABLES") & _set_literal("DEDUP_TABLES") + assert not both, ( + f"{sorted(both)} are both replaced on every batch and skipped as " + f"duplicates, which leaves them permanently empty after the second " + f"batch while the agent reports sending rows" + ) + + +def test_the_overlap_is_refused_at_import(): + """A startup failure rather than a tab that is empty for reasons nobody + can see from either side. This class of bug is invisible in production + precisely because every component reports success.""" + source = SERVER.read_text(encoding="utf-8") + assert "_assert_ingest_modes_are_exclusive()" in source, \ + "nothing enforces the invariant, so the next table added can break it" + + tree = ast.parse(source) + fn = next(n for n in ast.walk(tree) + if isinstance(n, ast.FunctionDef) + and n.name == "_assert_ingest_modes_are_exclusive") + assert "raise" in ast.unparse(fn) + + +def test_the_guard_actually_fires(): + """Compiled and run against an overlapping pair, because a guard that + cannot fail is decoration.""" + tree = ast.parse(SERVER.read_text(encoding="utf-8")) + fn = next(n for n in ast.walk(tree) + if isinstance(n, ast.FunctionDef) + and n.name == "_assert_ingest_modes_are_exclusive") + namespace = {"SNAPSHOT_TABLES": {"clash"}, "DEDUP_TABLES": {"clash"}} + exec(compile(ast.Module(body=[fn], type_ignores=[]), "", "exec"), + namespace) + with pytest.raises(RuntimeError) as info: + namespace["_assert_ingest_modes_are_exclusive"]() + assert "clash" in str(info.value) + + +def test_the_snapshot_list_is_not_written_out_twice(): + """It was a named set in one place and an inline literal in the ingest + loop. Two copies of the same list is how one of them came to disagree + with DEDUP_TABLES without anybody noticing.""" + source = SERVER.read_text(encoding="utf-8") + tree = ast.parse(source) + fn = next(n for n in ast.walk(tree) + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + and n.name == "insert_data") + code = ast.unparse(fn) + assert "SNAPSHOT_TABLES" in code + assert "'network_connections', 'hardware_inventory'" not in code + + +@pytest.mark.parametrize("table", [ + "network_connections", "hardware_inventory", "docker_containers", + "critical_files", "resource_usage", "disk_usage", +]) +def test_current_state_tables_are_snapshots(table): + """These describe what is true on the host right now. The agent re-sends + the whole picture on purpose, so treating a repeat as a duplicate throws + away the only copy.""" + assert table in _set_literal("SNAPSHOT_TABLES"), table + + +@pytest.mark.parametrize("table", [ + "siem_events", "events_alert", "fim_data", "packages", + "vulnerabilities_report", "portscan_result", +]) +def test_event_history_tables_are_deduplicated(table): + """These accumulate. Emptying one on every batch would discard the + history that is the whole reason to keep it.""" + assert table in _set_literal("DEDUP_TABLES"), table + + +def test_every_ingested_table_has_a_mode(): + """A table in neither set accumulates without deduplication, which is + almost never what anybody wanted - it is the default you get by + forgetting, and it grows without bound.""" + allowed = _set_literal("ALLOWED_TABLES") + classified = _set_literal("SNAPSHOT_TABLES") | _set_literal("DEDUP_TABLES") + unclassified = sorted(allowed - classified) + assert not unclassified, ( + f"{unclassified} are accepted from agents but are neither a snapshot " + f"nor deduplicated, so they accumulate every duplicate for ever" + ) diff --git a/tests/test_telemetry_health.py b/tests/test_telemetry_health.py new file mode 100644 index 0000000..6925ff7 --- /dev/null +++ b/tests/test_telemetry_health.py @@ -0,0 +1,222 @@ +"""Does the telemetry actually arrive? Per table, end to end. + +Three bugs in one day had the same shape, and none of them was visible from +either side alone: + + 1. `edr_enforcer.main` called five collectors as five bare statements, so the + first to raise took the four after it. Network and hardware went dark + together and nothing said so. + 2. Four tables were both snapshot and deduplicated: emptied on every batch, + then every row skipped as a duplicate. Full once, permanently empty after. + 3. The agent marks rows sent as soon as `sendall` returns. The ingest + protocol has no reply, so a server that accepts a batch and stores none of + it is indistinguishable from one that stored all of it. + +Every layer reported success. The only symptom was an empty table, which reads +exactly like a host with nothing to report - and on a security console those +two mean opposite things. + +The fix is not more logging on either side. It is putting the two numbers next +to each other: the agent says what it holds and believes it shipped, the server +says what it holds, and the difference names the broken link. `sent (50 rows)` +against a server table with zero rows is not ambiguous once somebody compares +them; the whole difficulty was that nobody ever did. +""" +from __future__ import annotations + +import ast +import pathlib + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +APP = ROOT / "app.py" +AGENT_MAIN = ROOT / "Sentora" / "main.py" +HEALTH = ROOT / "Sentora" / "modules" / "telemetry_health.py" + + +def _classifier(): + """`_classify_link`, compiled without importing app.py.""" + tree = ast.parse(APP.read_text(encoding="utf-8")) + fn = next(n for n in ast.walk(tree) + if isinstance(n, ast.FunctionDef) and n.name == "_classify_link") + namespace: dict = {} + exec(compile(ast.Module(body=[fn], type_ignores=[]), "", "exec"), + namespace) + return namespace["_classify_link"] + + +def _function(path: pathlib.Path, name: str) -> str: + tree = ast.parse(path.read_text(encoding="utf-8")) + return next(ast.unparse(n) for n in ast.walk(tree) + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + and n.name == name) + + +# -------------------------------------------------------------------------- +# The states, each of which used to look like an empty table +# -------------------------------------------------------------------------- + +def test_shipped_but_not_here_is_its_own_state(): + """The bug that was invisible. The agent's log said `sent (50 rows)` every + cycle for hours while the server held none of them.""" + state, detail = _classifier()( + {"held": 50, "unsent": 0, "shipped": 50}, 0) + assert state == "lost in transit" + assert "not a quiet host" in detail + + +def test_never_collected_is_not_the_same_as_lost(): + """A collector that never ran and a batch that was discarded need + different people to do different things, and both showed as a blank + table.""" + state, _ = _classifier()({"held": 0, "unsent": 0, "shipped": 0}, 0) + assert state == "not collected" + + +def test_a_send_failure_carries_its_reason(): + state, detail = _classifier()( + {"held": 10, "unsent": 10, "shipped": 0, + "last_error": "OSError: connection refused"}, 0) + assert state == "send failing" + assert "connection refused" in detail + + +def test_collected_but_not_yet_shipped_is_normal(): + """The ordinary state a few seconds after a collector runs. Reporting it + as broken would train people to ignore this page.""" + state, _ = _classifier()({"held": 10, "unsent": 10, "shipped": 0}, 0) + assert state == "queued" + + +def test_flowing_says_so(): + state, _ = _classifier()({"held": 50, "unsent": 0, "shipped": 50}, 50) + assert state == "flowing" + + +def test_a_missing_server_table_is_a_state_not_an_error(): + """The agent can be newer than the server. That is worth naming rather + than failing the whole report over.""" + state, _ = _classifier()({"held": 5, "unsent": 0, "shipped": 5}, None) + assert state == "no table" + + +def test_an_empty_but_healthy_table_is_not_flagged(): + """Plenty of tables are legitimately empty on a given host. If those read + as broken the page is noise and nobody opens it.""" + state, _ = _classifier()({"held": 0, "unsent": 0, "shipped": 0}, 0) + assert state != "lost in transit" + + +# -------------------------------------------------------------------------- +# The agent's half +# -------------------------------------------------------------------------- + +def test_the_agent_reads_counts_from_the_database_not_a_counter(): + """A counter resets when the agent restarts and drifts whenever anything + writes without going through it. The tables are the truth.""" + source = HEALTH.read_text(encoding="utf-8") + assert "SELECT COUNT(*)" in source + assert "FILTER (WHERE sent = FALSE)" in source + + +def test_the_agent_does_not_judge_its_own_delivery(): + """The comparison needs both halves, and the other half is on the server. + An agent deciding whether its own data arrived is the assumption that + produced this bug.""" + source = HEALTH.read_text(encoding="utf-8") + for verdict in ("lost in transit", "flowing", "not collected"): + assert verdict not in source, \ + f"the agent classifies delivery ({verdict!r}); only the server can" + + +def test_a_missing_table_does_not_hide_the_others(): + """The agent's schema gains tables over releases. One missing must not + take the whole report down with it.""" + code = _function(HEALTH, "_counts") + assert "except Exception" in code + # `ast.unparse` writes the tuple with parentheses. + assert "return (0, 0)" in code + + +def test_the_send_path_records_both_outcomes(): + code = _function(AGENT_MAIN, "send_table") + assert "telemetry_health.record_send(" in code + assert "record_send_failure" in code + + +def test_the_agent_serves_the_report_over_the_channel(): + code = _function(AGENT_MAIN, "dispatch_channel_request") + assert "/telemetry/health" in code + assert "telemetry_health.report" in code + + +def test_the_module_is_bundled_by_both_builds(): + """Reached only from `main.py`'s import block and `dispatch_channel_request`. + Left out of one build script it produces an agent that installs fine on one + platform and cannot answer this question.""" + for script in ("build_agent.ps1", "build_agent.sh"): + text = (ROOT / "Sentora" / script).read_text(encoding="utf-8") + assert "modules.telemetry_health" in text, script + + +# -------------------------------------------------------------------------- +# The server's half +# -------------------------------------------------------------------------- + +def test_an_agent_too_old_to_answer_is_told_apart_from_a_broken_one(): + """A 501 from the dispatcher means "this build does not implement it", + which is a different instruction from "something is wrong".""" + code = _function(APP, "get_telemetry_health") + assert "501" in code + assert "Rebuild and reinstall" in code + + +def test_the_report_counts_what_is_broken(): + """So the page and, later, an alert can act on one number rather than on + a table somebody has to read.""" + code = _function(APP, "get_telemetry_health") + assert "broken_count" in code + + +def test_the_server_counts_its_own_rows_rather_than_trusting_the_agent(): + """The entire point. Asking the agent whether its data arrived returns + the same optimistic answer that hid the bug.""" + code = _function(APP, "_server_table_counts") + assert "SELECT COUNT(*)" in code + + +# -------------------------------------------------------------------------- +# It has to be reachable from the page where the symptom appears +# -------------------------------------------------------------------------- + +FRONTEND = ROOT / "frontend" / "src" + + +def test_the_page_is_routed_and_linked(): + """A diagnosis nobody can navigate to is a diagnosis nobody will make. + The link belongs on the host page, because that is where the empty tab + that prompts the question is.""" + app_tsx = (FRONTEND / "App.tsx").read_text(encoding="utf-8") + assert "/agent/:agentName/telemetry" in app_tsx + + detail = (FRONTEND / "pages" / "AgentDetail.tsx").read_text(encoding="utf-8") + assert "/telemetry" in detail + + +def test_the_broken_rows_sort_first(): + """A page that needs scrolling to find the problem is a page nobody opens + twice.""" + page = (FRONTEND / "pages" / "TelemetryHealth.tsx").read_text(encoding="utf-8") + assert "STATE_RANK" in page + order = page[page.index("const STATE_RANK"):page.index("const rank")] + assert order.index("lost in transit") < order.index("flowing") + + +def test_an_unreachable_agent_does_not_render_as_a_healthy_one(): + """The ambiguity this whole feature exists to remove. If the fetch fails + and the page shows an empty table, it has reproduced the bug it + diagnoses.""" + page = (FRONTEND / "pages" / "TelemetryHealth.tsx").read_text(encoding="utf-8") + assert "ErrorState" in page + assert "Could not reach this agent" in page