app.py

from sanic import Request, Sanic
import bcrypt
import aiomysql
import inspect
import secrets
import os, pathlib
import stat
import time
from enum import Enum
import re
from sanic_cors import CORS
from sanic.response import json, file, HTTPResponse, raw
import multiprocessing
import json as pyjson
from mysql.connector.aio import connect 
import mysql.connector
from datetime import datetime, timezone
from time import sleep
import core.opensearch as os_utils
import smtplib
from email.message import EmailMessage
from typing import List, Any
from pydantic import BaseModel, EmailStr, constr, Field, field_validator
from typing import Optional, List

class LoginRequest(BaseModel):
    # The same pattern CreateUserRequest has always had. Login did not, which
    # mattered because the LDAP path interpolates this straight into a search
    # filter: `*)(uid=*` turns an authentication check into a wildcard match.
    # A username that cannot be created is not a username that should be
    # accepted at login either.
    # Hyphen last, so it is a literal rather than a range and needs no escape.
    username: constr(min_length=3, max_length=50, pattern=r'^[a-zA-Z0-9_.@-]+$')
    password: constr(min_length=6)

class CreateUserRequest(BaseModel):
    username: constr(min_length=3, max_length=50, pattern=r'^[a-zA-Z0-9_-]+$')
    password: constr(min_length=8)
    role: str = "user"

    @field_validator('password')
    @classmethod
    def password_complexity(cls, v: str):
        if not re.search(r"[A-Z]", v):
            raise ValueError('Password must contain at least one uppercase letter')
        if not re.search(r"[0-9]", v):
            raise ValueError('Password must contain at least one number')
        return v

class ChangePasswordRequest(BaseModel):
    username: str
    current_password: str
    new_password: constr(min_length=8)

    @field_validator('new_password')
    @classmethod
    def password_complexity(cls, v: str):
        if not re.search(r"[A-Z]", v):
            raise ValueError('Password must contain at least one uppercase letter')
        if not re.search(r"[0-9]", v):
            raise ValueError('Password must contain at least one number')
        return v

import asyncio
import hashlib
import requests
import textwrap
from sanic.exceptions import SanicException
import subprocess
from functools import wraps
from sanic.response import json as sanic_json
import ldap3
from sanic import response
from ldap3 import Server, Connection, ALL, Tls
from ldap3.utils.conv import escape_filter_chars
from ldap3.core.exceptions import LDAPSocketOpenError, LDAPBindError
import ssl
from cryptography.fernet import Fernet, InvalidToken
from dotenv import load_dotenv, set_key, dotenv_values
 
ENC_PREFIX = "enc::"
ENV_PATH = pathlib.Path(".env")

if ENV_PATH.exists():
    load_dotenv(dotenv_path=ENV_PATH)

CORS_ORIGINS = [o.strip() for o in os.getenv("CORS_ORIGINS", "").split(",") if o.strip()]
# A credentialed request cannot use a wildcard origin: the browser rejects
# `Access-Control-Allow-Origin: *` outright whenever cookies are attached. In
# the normal deployment this app serves the SPA itself, so requests are
# same-origin and no CORS entry is needed at all — hence the empty default.
if "*" in CORS_ORIGINS:
    print("[!] CORS_ORIGINS='*' is incompatible with cookie authentication and was ignored. "
          "List explicit origins if the UI is served from a different host.")
    CORS_ORIGINS = [o for o in CORS_ORIGINS if o != "*"]
DB_HOST = os.getenv("DB_HOST", "127.0.0.1")
DB_USER = os.getenv("DB_USER", "root")
DB_PASSWORD = os.getenv("DB_PASSWORD", "my-secret-pw")
DB_PORT = int(os.getenv("DB_PORT", "3306"))

multiprocessing.freeze_support()
app = Sanic("SIEMLoggerAPI")
app.config.RESPONSE_TIMEOUT = 600
app.config.REQUEST_TIMEOUT = 600
app.config.WORKER_ACK_TIMEOUT = 60.0
CORS(app, origins=CORS_ORIGINS, supports_credentials=True)

app.static("/assets", "./frontend/dist/assets", name="frontend_assets")
app.static("/vite.svg", "./frontend/dist/vite.svg", name="frontend_logo")

from ai.utils import load_ai_config, is_critical_log, save_ai_results
from security import session as session_store
from security import ssrf
from security import totp as totp_store
from security import webauthn as webauthn_store
from core import agent_link
from core import attack
from core import config_validation
from core import login_guard
from core import netloc
from core import self_defence
from core import threat_feeds
from core import tls as product_tls
from core import version as product_version
# Installer templating: pure string generation, extracted so app.py is not
# also 420 lines of PowerShell. See core/installers.py.
from core.installers import _render_linux_install, _render_windows_install

# Cookies are only marked Secure when the platform is actually served over
# TLS; a Secure cookie on a plain-http lab deployment is silently dropped by
# the browser and nobody can log in. Set SESSION_COOKIE_SECURE=1 in .env once
# you terminate TLS in front of the app.
#
# Unset now means "follow TLS_ENABLED" rather than "0". The pair had to agree
# and nothing made them: an operator who turned on TLS and left this alone got
# an HTTPS console whose session cookie was still allowed onto plain HTTP.
# An explicit value still wins, because TLS terminating at a proxy is
# invisible from in here and that deployment needs the flag by hand.
SESSION_COOKIE_SECURE = product_tls.cookie_should_be_secure()

# 'Lax' blocks the cookie on cross-site POST/XHR, which is the CSRF case that
# matters, while staying compatible with normal same-origin use. 'Strict' is
# available for deployments that want it; 'None' requires Secure and exists
# only for split-origin setups.
SESSION_COOKIE_SAMESITE = os.getenv("SESSION_COOKIE_SAMESITE", "Lax").strip().capitalize()
if SESSION_COOKIE_SAMESITE not in ("Lax", "Strict", "None"):
    print(f"[!] Invalid SESSION_COOKIE_SAMESITE={SESSION_COOKIE_SAMESITE!r}; falling back to 'Lax'.")
    SESSION_COOKIE_SAMESITE = "Lax"

AGENT_SHARED_SECRET = os.getenv("AGENT_SHARED_SECRET", "") or os.getenv("AGENT_SHARED_SECRET", "")

OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://ollama:11434/api")
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "llama3.2:3b")

if not AGENT_SHARED_SECRET:
    AGENT_SHARED_SECRET = secrets.token_urlsafe(32)
    print("[+] Generated ephemeral agent shared secret (set AGENT_SHARED_SECRET in .env to override).")

# Fields to attempt decryption on. Deliberately a SUPERSET of what the agent
# currently encrypts (see Sentora/modules/enc_db.ENCRYPT_FIELDS_MAP).
#
# That asymmetry is safe and intentional: decrypt_row_fields skips any value
# that does not carry the `enc::` prefix, so listing a plaintext column costs
# nothing. Trimming this list to match the agent exactly would be the unsafe
# direction — older agents encrypted a different set, and a field dropped from
# here would render as raw ciphertext for any row written back then.
#
# Add to this map when the agent starts encrypting something new. Do not
# remove from it.
ENCRYPTED_FIELDS_MAP = {
    "siem_events": ["source", "timestamp", "message"],
    "critical_files": ["path", "owner", "grp", "permissions", "last_opened"],
    "packages": ["package", "version"],
    "vulnerabilities_report": ["package_name", "package_version", "vulnerability_id", "summary", "details_url"],
    "events_alert": ["source", "timestamp", "severity", "score", "categories", "message"],
    "soar_actions": ["timestamp", "action", "target", "comment", "status"],
    "fim_data": ["path", "hash_sha256"],
    "registry_logs": ["hive", "key_path", "value_name", "value_data"],
    "network_connections": ["process_name", "local_addr", "remote_addr"],
    "process_events": ["name", "cmdline", "username"],
    "hardware_inventory": ["name", "serial_number"],
    "security_audit": ["finding", "details"]
}


# The five boot migrations moved to core/schema_init.py: they are DDL, they
# share one dependency, and app.py is not where a schema belongs.
from core import schema_init  # noqa: E402


@app.before_server_start
async def setup_hub(app):
    """Create and migrate the schema the platform cannot run without.

    `before_server_start`, not `main_process_start`. That listener only fires
    when Sanic runs a multi-process manager, and `app.run(single_process=True)`
    - which is what `WORKERS=1` selects, and what the compose file asks for -
    has no such manager. So nothing here ran at all: no `users`, no
    `sessions`, no enrolment tables.

    The symptom was three layers away from the cause. Password checks go
    through the synchronous connector and kept working, so `/login`
    authenticated the operator, failed to write a session into a table that
    did not exist, and the handler's own `except` read that as "not a local
    user" and answered 401. A correct password, told it was wrong, because a
    startup hook was registered for a process model this server does not use.

    Every migration is idempotent and cheap on a database that is already
    correct, so running it per worker rather than once is not worth a second
    process model to arrange.
    """
    schema_init.set_defaults(conn_factory=sync_mysql_conn,
                             alert_template=DEFAULT_ALERT_TEMPLATE)
    await schema_init.run_all()


def is_soar_enabled() -> bool:
    """Community edition: SOAR is unconditionally enabled."""
    return True


def _soar_block_response():
    """Legacy guard return — never fires in Community edition because
    is_soar_enabled() always returns True. Kept so existing call sites that
    `if not is_soar_enabled(): return _soar_block_response()` stay valid."""
    return sanic_json(
        {"status": "error", "message": "SOAR unavailable"},
        status=503,
    )


AUTOMATION_ALLOWED_ACTIONS = {
    "block_ip",
    "unblock_ip",
    "disable_user",
    "enable_user",
    "kill_process",
    "restart_service",
    "lock_machine",
    "quarantine_file",
    "tail_log",
    "run_cmd",
    "flush_dns",
    "disable_interface",
    "logoff_user",
    "clear_temp",
    "dump_process",
    "container_kill",
    "container_stop",
    "container_isolate",
    "suspend_process",
    "delete_registry_key",
    "protect_shadows",
    "start_vnc",
    "stop_vnc"
}
AUTOMATION_ALLOWED_STATUSES = {"pending", "active", "paused", "completed", "failed"}

SOAR_TO_AUTO_STATUS = {
    "success": "completed",
    "permanent": "completed",
    "resolved": "completed",
    "resolve_failed": "failed",
    "failed": "failed",
}


def _is_valid_ipv4(ip: str) -> bool:
    parts = str(ip).split(".")
    if len(parts) != 4: return False
    try:
        return all(0 <= int(p) <= 255 for p in parts)
    except Exception:
        return False

_USER_RE = re.compile(r"^[A-Za-z0-9_.-]{1,32}$")
def _is_valid_username(u: str) -> bool:
    return bool(_USER_RE.match(str(u or "")))

def ensure_permissions(path: pathlib.Path) -> bool:
    """Restrict a secret file to its owner, and say so if that did not happen.

    This swallowed the failure. When chmod does not take effect the file keeps
    whatever the umask gave it - commonly 0644 in a container - so the secret
    is readable by every process there and nothing anywhere says so. A
    permission control that fails silently is not a control.

    Verified rather than assumed: chmod can return without error and still not
    produce the mode asked for, on Windows and on some mounted filesystems.

    Returns True when the file ends up owner-only. Never raises: the caller
    has already written the secret, and refusing to continue would leave it
    written and unusable.
    """
    try:
        os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
    except OSError as e:
        print(f"[!] Could not restrict permissions on {path}: {e}. "
              f"The file may be readable by other users.", flush=True)
        return False

    # POSIX modes are advisory on Windows, where this is expected to be a
    # no-op; there is no point reporting it on every start.
    if os.name == "nt":
        return True
    try:
        mode = stat.S_IMODE(os.stat(path).st_mode)
    except OSError:
        return False
    if mode & (stat.S_IRWXG | stat.S_IRWXO):
        print(f"[!] {path} is still mode {mode:o} after chmod — it is "
              f"readable beyond its owner.", flush=True)
        return False
    return True


def _as_bool(val):
    if isinstance(val, bool):
        return val
    if isinstance(val, (int, float)):
        return val == 1
    if isinstance(val, str):
        return val.strip().lower() in {"1", "true", "yes", "y", "on", "active"}
    return False

def load_or_create_fernet_from_env():
    """Resolve the server-side Fernet key, which protects at-rest columns.

    Normally supplied as `FERNET_KEY` in the environment (compose passes it
    through from .env). The fallback generates one and writes it back to .env
    so it survives a restart — but the container runs as a non-root user and
    /app is not writable, so that path only works for a bare-metal dev run.

    Generating a key we cannot persist would be the worst outcome: every
    restart would produce a different key and silently render previously
    encrypted values unreadable. So a failed write is fatal and says exactly
    what to do about it.
    """
    key_b64 = os.getenv("FERNET_KEY")
    if key_b64:
        return Fernet(key_b64.encode())

    vals = dotenv_values(dotenv_path=ENV_PATH) if ENV_PATH.exists() else {}

    key_b64 = vals.get("FERNET_KEY")
    if not key_b64:
        key_b64 = Fernet.generate_key().decode()
        try:
            if not ENV_PATH.exists():
                ENV_PATH.touch()
            set_key(str(ENV_PATH), "FERNET_KEY", key_b64)
            ensure_permissions(ENV_PATH)
            print(f"[+] Generated FERNET_KEY and persisted it to {ENV_PATH.resolve()}")
        except OSError as e:
            raise SystemExit(
                f"[FATAL] No FERNET_KEY set and it could not be persisted to "
                f"{ENV_PATH} ({e}).\n"
                f"        Add a key to your .env — compose passes it through:\n"
                f"          FERNET_KEY={key_b64}\n"
                f"        Losing this key makes existing encrypted columns "
                f"unreadable, so store it somewhere you can restore from."
            ) from e

    return Fernet(key_b64.encode())

fernet = load_or_create_fernet_from_env()

import zipfile
import io

@app.get("/api/agent/ca")
async def download_ca(request):
    """The self-signed CA certificate, for agents that have to verify this
    server.

    Public, and safe to be. This serves `rootCA.crt` and never `rootCA.key`:
    the certificate is the half of the pair whose entire purpose is to be
    distributed, and publishing it grants nothing - it lets a client check a
    signature it could not otherwise check.

    Fetching it over the connection it is meant to secure is trust on first
    use, and worth being honest about rather than dressing up. It is the same
    trust the enrolment token and the agent binary already travel on in the
    same installer run: anyone able to intercept this exchange can hand over a
    binary of their choosing, and the CA is the smallest of those problems.
    Deployments that cannot accept that should ship `rootCA.crt` out of band
    and set `server_ca` by hand.

    404 rather than an empty body when there is no CA, because a zero-byte
    PEM written to disk fails later with a message about the certificate
    format, at a point far from the cause.
    """
    # Resolved through `core.tls`, not from a path written out again here. The
    # directory moves with TLS_DIR, and a second copy of the path is how the
    # route that hands out the CA comes to look somewhere the generator never
    # wrote to.
    ca = product_tls.ca_certificate()
    if ca is None:
        return json({
            "error": "no_local_ca",
            "message": "This server has no self-signed CA. That is expected "
                       "when TLS is off, or when TLS_CERT points at a "
                       "certificate from a real CA - in which case the agent "
                       "needs no extra trust and server_ca can stay unset.",
        }, status=404)
    return await response.file(
        str(ca), mime_type="application/x-pem-file",
        filename="rootCA.crt")


@app.get("/api/agent/download/<os_type>")
async def download_agent(request, os_type):
    user_id = current_user_id(request)
    provided_key = request.headers.get("X-Agent-Key") or request.args.get("key")
    agent_key = request.headers.get("X-Agent-Key") or request.args.get("agent_key")

    is_authed = False
    if user_id and await user_has_permission(user_id, "manage_agent"):
        is_authed = True
    elif provided_key == AGENT_SHARED_SECRET:
        is_authed = True
    elif agent_key:
        try:
            with sync_mysql_conn("userdb") as conn:
                cur = conn.cursor()
                try:
                    cur.execute(
                        "SELECT 1 FROM agent_identities WHERE agent_key=%s AND revoked_at IS NULL",
                        (agent_key,)
                    )
                    if cur.fetchone():
                        is_authed = True
                finally:
                    cur.close()
        except Exception:
            pass

    if not is_authed:
        return sanic_json({"status": "error", "message": "Unauthorized"}, status=403)

    await audit_log(request, "DOWNLOAD_AGENT", os_type, f"Agent package requested for {os_type}")

    server_ip = request.host.split(':')[0]
    
    memory_file = io.BytesIO()
    
    agent_root = pathlib.Path("Sentora")
    if not agent_root.exists():
        return sanic_json({"ok": False, "error": "Agent source not found"}, status=404)

    exe_name = "main.exe" if os_type == "windows" else "main"

    bin_path = agent_root / exe_name
    if not bin_path.exists():
        return sanic_json({
            "ok": False,
            "error": f"Agent binary '{exe_name}' is not built. "
                     f"Run Sentora/build_agent.{'ps1' if os_type=='windows' else 'sh'} "
                     f"to produce it before enrolling agents."
        }, status=503)

    files_to_zip = [
        exe_name,
        "docker-compose.yml",
        "db/init.sql"
    ]

    with zipfile.ZipFile(memory_file, 'w', zipfile.ZIP_DEFLATED) as zf:
        for f_path in files_to_zip:
            abs_path = agent_root / f_path
            if abs_path.exists():
                zf.write(abs_path, f_path)

        if os_type == "windows":
            setup_bat = f"""@echo off
setlocal
cd /d "%~dp0"

:: Check for Administrator privileges
openfiles >nul 2>&1
if %errorlevel% neq 0 (
    echo [!] This script requires Administrator privileges.
    echo [*] Attempting to elevate...
    powershell.exe -Command "Start-Process '%~f0' -Verb RunAs"
    exit /b
)

echo [*] Starting Sentora Agent Database (Docker)...
if exist "docker-compose.yml" (
    docker compose up -d
)

echo [*] Creating Background Runner...
echo @echo off > run_agent.bat
echo cd /d "%%~dp0" >> run_agent.bat
:: Prevent Python from trying to format %%COMPUTERNAME%%, just let batch handle it
echo "main.exe" -a "Agent-%%COMPUTERNAME%%" -s "{server_ip}" >> run_agent.bat

echo [*] Installing Sentora Agent Background Task...
set TASK_NAME=SentoraAgent
set WATCH_NAME=SentoraAgentWatchdog
set BIN_PATH="%~dp0run_agent.bat"

:: Stop and delete old Scheduled Tasks if they exist
schtasks /end /tn "%TASK_NAME%" >nul 2>&1
schtasks /delete /tn "%TASK_NAME%" /f >nul 2>&1
schtasks /delete /tn "%WATCH_NAME%" /f >nul 2>&1

:: Run silently on boot as SYSTEM
schtasks /create /tn "%TASK_NAME%" /tr "%BIN_PATH%" /sc onstart /ru System /f
if %errorlevel% neq 0 (
    echo [!] Failed to create background task.
) else (
    echo [*] Starting the Agent now...
    schtasks /run /tn "%TASK_NAME%"
)

:: Watchdog. schtasks cannot attach a repetition interval to an onstart
:: trigger, so this is a second task on a 15-minute schedule. main.exe takes
:: a single-instance lock before doing any work and exits 0 if another agent
:: already holds it -- so while the agent is healthy this is a no-op, and
:: when it has died it brings it back without waiting for a reboot.
schtasks /create /tn "%WATCH_NAME%" /tr "%BIN_PATH%" /sc minute /mo 15 /ru System /f >nul 2>&1
if %errorlevel% neq 0 (
    echo [!] Watchdog task not created - the agent will still start at boot.
) else (
    echo [*] Watchdog registered ^(checks every 15 minutes^).
)

echo [+] Setup Complete! The Agent is now running in the background.
pause
"""
            zf.writestr("setup.bat", setup_bat)
        else:
            setup_sh = f"""#!/bin/bash
cd "$(dirname "$0")"

if [ "$EUID" -ne 0 ]; then
  echo "[!] Please run as root (sudo ./setup.sh)"
  exit 1
fi

echo "[*] Starting Sentora Agent Database (Docker)..."
if command -v docker >/dev/null 2>&1; then
    docker compose up -d || docker-compose up -d
else
    echo "[!] Docker is not installed. Database cannot be started."
fi

echo "[*] Installing Sentora Agent Service..."
chmod +x main

SERVICE_FILE="/etc/systemd/system/sentora-agent.service"
cat > $SERVICE_FILE << EOF
[Unit]
Description=Sentora Agent
After=network.target docker.service

[Service]
ExecStart=$(pwd)/main -a "Agent-$(hostname)" -s "{server_ip}"
WorkingDirectory=$(pwd)
Restart=always
User=root

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable sentora-agent
systemctl start sentora-agent

echo "[+] Setup Complete! The Agent is now running in the background."
"""
            zf.writestr("setup.sh", setup_sh)

    memory_file.seek(0)

    return HTTPResponse(
        body=memory_file.getvalue(),
        content_type="application/zip",
        headers={
            "Content-Disposition": f"attachment; filename=Sentora-Agent-{os_type}.zip",
            "Access-Control-Expose-Headers": "Content-Disposition"
        }
    )

ENROLLMENT_TOKEN_TTL_HOURS = int(os.getenv("ENROLLMENT_TOKEN_TTL_HOURS", "24"))

def _gen_token() -> str:
    return secrets.token_hex(32)


def _quote_identifier(name: str) -> str:
    """Quote a table or database name for interpolation into SQL.

    Identifiers cannot be parameterised - `DESCRIBE %s` is not valid SQL - so
    they have to be interpolated, and `f"DESCRIBE `{table}`"` was doing it
    without escaping. A name containing a backtick closes the quoting early:
    ``tbl`; DROP DATABASE x; -- `` becomes two statements.

    MySQL escapes a backtick inside an identifier by doubling it. The name is
    also length-limited and rejected outright if it contains a NUL or a
    newline, neither of which appears in a real identifier and both of which
    make log entries misleading.

    These paths all require `manage_db`, so this is depth rather than the only
    control.
    """
    raw = str(name or "")
    if not raw or len(raw) > 64 or any(bad in raw for bad in ("\x00", "\n", "\r")):
        raise ValueError(f"not a usable SQL identifier: {raw[:40]!r}")
    return "`" + raw.replace("`", "``") + "`"


def _client_ip(request) -> str:
    """The client's address, per core/login_guard.client_ip.

    This returned X-Forwarded-For unconditionally. With no reverse proxy in
    front the header is whatever the caller typed, so every address in the
    audit trail was attacker-choosable - log injection into the SIEM's own
    audit log - and any per-address limit built on it resets itself.
    """
    return login_guard.client_ip(request)

async def _validate_agent_auth(request) -> str | None:
    """Return agent_name if X-Agent-Key matches a non-revoked identity, else None.
    Also accepts the global AGENT_SHARED_SECRET for backward compat (returns '*')."""
    key = request.headers.get("X-Agent-Key") or request.args.get("agent_key")
    if not key:
        legacy = request.headers.get("X-Agent-Key") or request.args.get("key")
        if legacy and legacy == AGENT_SHARED_SECRET:
            return "*"
        return None
    try:
        with sync_mysql_conn("userdb") as conn:
            cur = conn.cursor()
            try:
                cur.execute(
                    "SELECT agent_name FROM agent_identities WHERE agent_key=%s AND revoked_at IS NULL",
                    (key,)
                )
                row = cur.fetchone()
                if row:
                    return row[0]
            finally:
                cur.close()
    except Exception as e:
        print(f"[Enroll] agent auth lookup error: {e}")
    if key == AGENT_SHARED_SECRET:
        return "*"
    return None

async def _require_agent(request, agent: str):
    """Authorise an agent-facing call, returning an error response or None.

    The automation-poll pair used to carry no authentication at all, and
    `report_automation_result_by_id` took the agent name from the request
    body. That gave anyone who could reach the API two primitives:

      GET  /<agent>/automations/pending
          read the response actions queued for a host - an attacker sees
          `ISOLATE_HOST` for their own machine before the agent does.

      POST /<agent>/automations/report {"task_id": N, "status": "SUCCESS"}
          walk task_id from 1 upward marking everything done. The real agent
          then never sees those rows, because it polls WHERE status='pending'.
          The action never runs and the console shows it green.

    The second is worse than an authentication bypass. It is a way to switch
    off every autonomous response the platform makes while it continues to
    report that they succeeded.

    Identity comes from the key, never from the URL or the body, and it has to
    match the agent being acted on.
    """
    identity = await _validate_agent_auth(request)
    if identity is None:
        return sanic_json({"status": "error", "message": "unauthorized"},
                          status=401)
    # "*" is the legacy fleet-wide secret. Kept working, because agents
    # enrolled before per-agent keys existed still use it, but it is not an
    # identity - it cannot be used to speak for a specific host beyond what it
    # already could.
    if identity == "*":
        return None
    if _agent_name_forms(identity)[1] != _agent_name_forms(agent)[1]:
        print(f"[!] agent key for {identity!r} used against {agent!r}", flush=True)
        return sanic_json({"status": "error", "message": "unauthorized"},
                          status=401)
    return None


@app.post("/api/agents/enroll")
async def enroll_agent(request):
    """Create a one-time enrollment token (authenticated, requires manage_agent)."""
    user_id = current_user_id(request)
    if not user_id or not await user_has_permission(user_id, "manage_agent"):
        return sanic_json({"status": "error", "message": "Unauthorized"}, status=403)

    body = request.json or {}
    hostname_hint = (body.get("hostname_hint") or "").strip()[:255]
    note = (body.get("note") or "").strip()[:500]
    ttl_hours = int(body.get("ttl_hours") or ENROLLMENT_TOKEN_TTL_HOURS)
    ttl_hours = max(1, min(ttl_hours, 24 * 30))

    username = None
    try:
        with sync_mysql_conn("userdb") as conn:
            cur = conn.cursor()
            try:
                cur.execute("SELECT username FROM users WHERE id=%s", (user_id,))
                r = cur.fetchone()
                if r:
                    username = r[0]
            finally:
                cur.close()
    except Exception:
        pass

    token = _gen_token()
    expires_at = datetime.now().replace(microsecond=0)
    from datetime import timedelta as _td
    expires_at = expires_at + _td(hours=ttl_hours)

    try:
        with sync_mysql_conn("userdb") as conn:
            cur = conn.cursor()
            try:
                cur.execute(
                    """INSERT INTO enrollment_tokens
                       (token, created_by_user_id, created_by_username, hostname_hint, note, expires_at)
                       VALUES (%s,%s,%s,%s,%s,%s)""",
                    (token, user_id, username, hostname_hint or None, note or None, expires_at)
                )
                conn.commit()
            finally:
                cur.close()
    except Exception as e:
        return sanic_json({"status": "error", "message": f"Failed to create token: {e}"}, status=500)

    await audit_log(request, "ENROLL_TOKEN_CREATE", token[:8] + "...",
                    f"ttl={ttl_hours}h hostname_hint={hostname_hint or '-'}")

    proto = "https" if request.scheme == "https" else "http"
    host = request.host
    base = f"{proto}://{host}"

    # A command that cannot work is worse than no command.
    #
    # These were emitted unconditionally, and on a self-signed deployment
    # neither one can fetch the script it is supposed to run:
    #
    #   iwr : Temel alinan baglanti kapatildi: SSL/TLS guvenli kanali icin
    #         guven iliskisi kurulamadi.
    #
    # Nothing on the endpoint trusts this CA yet - the installer is what puts
    # it there - so enrolment cannot begin. The console handed out a broken
    # one-liner and said nothing about why.
    #
    # So when this server signs with its own CA, the snippet fetches the
    # script with certificate validation suspended for that one request. That
    # is the same trust on first use the enrolment token in this URL already
    # depends on; the script it fetches installs the CA properly, and
    # everything after that verifies.
    #
    # `curl.exe` on Windows, not Invoke-WebRequest with a validation callback.
    # The obvious form,
    #
    #   [Net.ServicePointManager]::ServerCertificateValidationCallback = {$true}
    #
    # is a PowerShell script block, and .NET invokes it on an I/O thread with
    # no runspace. The callback itself throws, and what reaches the operator
    # is "an unexpected error occurred on a send" - which reads as a TLS
    # failure and sends you looking at protocol versions and cipher suites.
    # It cost an hour of exactly that. `curl.exe` ships with Windows 10 1803
    # and later, runs out of process, and leaves no static state to restore.
    self_signed = product_tls.ca_certificate() is not None
    linux = f"curl -fsSL '{base}/api/agent/deploy/linux?token={token}' | sudo bash"
    windows = f"iwr -useb '{base}/api/agent/deploy/windows?token={token}' | iex"
    if self_signed and proto == "https":
        linux = (f"curl -fsSLk '{base}/api/agent/deploy/linux?token={token}' "
                 f"| sudo bash")
        windows = (
            f"iex (curl.exe -fsSk "
            f"'{base}/api/agent/deploy/windows?token={token}' | Out-String)"
        )

    return sanic_json({
        "status": "success",
        "token": token,
        "expires_at": expires_at.strftime("%Y-%m-%d %H:%M:%S"),
        "server_url": base,
        # Whether the snippets carry a first-contact exception, so the UI can
        # say why rather than leaving somebody to wonder what the extra line
        # is doing.
        "self_signed": self_signed and proto == "https",
        "install": {
            "linux": linux,
            "windows": windows,
        }
    })

@app.get("/api/agents/enrollments")
async def list_enrollments(request):
    """List recent enrollment tokens (authed)."""
    user_id = current_user_id(request)
    if not user_id or not await user_has_permission(user_id, "manage_agent"):
        return sanic_json({"status": "error", "message": "Unauthorized"}, status=403)

    try:
        with sync_mysql_conn("userdb") as conn:
            cur = conn.cursor(dictionary=True)
            try:
                cur.execute(
                    """SELECT id, token, created_by_username, hostname_hint, note,
                              created_at, expires_at, used_at, used_by_agent, used_from_ip
                       FROM enrollment_tokens
                       ORDER BY created_at DESC LIMIT 100"""
                )
                rows = cur.fetchall()
            finally:
                cur.close()
        for r in rows:
            for k, v in list(r.items()):
                if isinstance(v, datetime):
                    r[k] = v.strftime("%Y-%m-%d %H:%M:%S")
            if r.get("token"):
                r["token_preview"] = r["token"][:8] + "…" + r["token"][-4:]
                del r["token"]
        return sanic_json({"status": "success", "enrollments": rows})
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

@app.delete("/api/agents/enrollments/<token_id:int>")
async def revoke_enrollment(request, token_id):
    user_id = current_user_id(request)
    if not user_id or not await user_has_permission(user_id, "manage_agent"):
        return sanic_json({"status": "error", "message": "Unauthorized"}, status=403)
    try:
        with sync_mysql_conn("userdb") as conn:
            cur = conn.cursor()
            try:
                cur.execute("DELETE FROM enrollment_tokens WHERE id=%s AND used_at IS NULL", (token_id,))
                conn.commit()
                affected = cur.rowcount
            finally:
                cur.close()
        if affected:
            await audit_log(request, "ENROLL_TOKEN_REVOKE", str(token_id), "")
            return sanic_json({"status": "success"})
        return sanic_json({"status": "error", "message": "Token not found or already used"}, status=404)
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

@app.post("/api/agents/register")
async def register_agent(request):
    """Exchange a one-time enrollment token for a per-agent identity.
    Body: {token, agent_name?, hostname?, os_type?}. Returns {agent_name, agent_key, server_url}."""
    body = request.json or {}
    token = (body.get("token") or "").strip()
    if not token or len(token) != 64:
        return sanic_json({"status": "error", "message": "Invalid token format"}, status=400)

    client_ip = _client_ip(request)
    hostname = (body.get("hostname") or "").strip()[:255] or None
    os_type = (body.get("os_type") or "").strip()[:32] or None
    req_name = (body.get("agent_name") or "").strip()[:128] or None
    # An upgrading agent sends the identity it already holds. Possession of
    # the key is the proof, so this cannot be used to claim someone else's
    # name; without a matching key the request falls through to a normal
    # fresh enrolment below.
    existing_key = (body.get("agent_key") or "").strip()[:128] or None

    if req_name and not re.match(r"^[A-Za-z0-9_.-]{1,128}$", req_name):
        return sanic_json({"status": "error", "message": "Invalid agent_name"}, status=400)

    try:
        with sync_mysql_conn("userdb") as conn:
            cur = conn.cursor(dictionary=True)
            try:
                cur.execute(
                    """SELECT id, expires_at, used_at FROM enrollment_tokens
                       WHERE token=%s FOR UPDATE""",
                    (token,)
                )
                row = cur.fetchone()
                if not row:
                    return sanic_json({"status": "error", "message": "Token not recognized"}, status=404)
                if row["used_at"] is not None:
                    # A one-time credential presented a second time. The
                    # ordinary cause is an installer re-run, and the one that
                    # matters is a token that leaked - they are the same
                    # request, so this is recorded rather than judged.
                    await _platform_event(
                        "ENROLMENT_TOKEN_REUSED", subject=token[:8] + "...",
                        source_ip=_client_ip(request),
                        detail=f"first used at {row['used_at']}")
                    return sanic_json({"status": "error", "message": "Token already used"}, status=409)
                if row["expires_at"] and row["expires_at"] < datetime.now():
                    return sanic_json({"status": "error", "message": "Token expired"}, status=410)

                # Re-enrolment of a known agent: the installer took the
                # upgrade path and is closing out the token it was given.
                # Without this the token stayed unused forever and the
                # Deploy page reported "waiting" after a deployment that had
                # in fact succeeded - the UI contradicting reality.
                if req_name and existing_key:
                    cur.execute(
                        "SELECT agent_name FROM agent_identities "
                        "WHERE agent_name=%s AND agent_key=%s",
                        (req_name, existing_key),
                    )
                    if cur.fetchone():
                        cur.execute(
                            """UPDATE enrollment_tokens
                               SET used_at=NOW(), used_by_agent=%s, used_from_ip=%s
                               WHERE id=%s""",
                            (req_name, client_ip, row["id"]),
                        )
                        conn.commit()
                        await audit_log(request, "AGENT_UPGRADE", req_name,
                                        f"existing identity kept, from {client_ip}")
                        proto = "https" if request.scheme == "https" else "http"
                        return sanic_json({
                            "status": "success",
                            "agent_name": req_name,
                            "agent_key": existing_key,
                            "server_url": f"{proto}://{request.host}",
                            "server_ip": request.host.split(":")[0],
                            "upgraded": True,
                        })
                    # Key did not match. Not an error worth failing on - fall
                    # through and enrol as a new agent, which is what a
                    # machine restored from a stale backup should get.

                if not req_name:
                    base = hostname or f"Agent-{token[:8]}"
                    req_name = re.sub(r"[^A-Za-z0-9_.-]", "-", base)[:128] or f"Agent-{token[:8]}"

                final_name = req_name
                suffix = 1
                while True:
                    cur.execute("SELECT 1 FROM agent_identities WHERE agent_name=%s", (final_name,))
                    if not cur.fetchone():
                        break
                    suffix += 1
                    final_name = f"{req_name}-{suffix}"[:128]
                    if suffix > 500:
                        return sanic_json({"status": "error", "message": "Could not allocate agent name"}, status=500)

                agent_key = _gen_token()
                cur.execute(
                    """INSERT INTO agent_identities
                       (agent_name, agent_key, os_type, hostname, enrolled_from_ip, enrolled_via_token)
                       VALUES (%s,%s,%s,%s,%s,%s)""",
                    (final_name, agent_key, os_type, hostname, client_ip, token)
                )
                cur.execute(
                    """UPDATE enrollment_tokens
                       SET used_at=NOW(), used_by_agent=%s, used_from_ip=%s
                       WHERE id=%s""",
                    (final_name, client_ip, row["id"])
                )
                conn.commit()
            finally:
                cur.close()
    except mysql.connector.IntegrityError as e:
        return sanic_json({"status": "error", "message": f"Identity conflict: {e}"}, status=409)
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

    proto = "https" if request.scheme == "https" else "http"
    host = request.host
    server_url = f"{proto}://{host}"
    server_ip = host.split(":")[0]
    return sanic_json({
        "status": "success",
        "agent_name": final_name,
        "agent_key": agent_key,
        "server_url": server_url,
        "server_ip": server_ip,
    })


@app.get("/api/agents/bootstrap")
async def agent_bootstrap(request):
    """Hand the shared Fernet key to a token-enrolled agent.

    Community edition: the key is read from the server's local file
    (FERNET_KEY_PATH) — no remote auth round-trip. The agent
    authenticates with the per-agent X-Agent-Key issued by
    /api/agents/register.
    """
    agent_key = (request.headers.get("X-Agent-Key") or "").strip()
    if not agent_key or len(agent_key) != 64:
        return sanic_json({"ok": False, "error": "missing X-Agent-Key"}, status=401)

    try:
        with sync_mysql_conn("userdb") as conn:
            cur = conn.cursor(dictionary=True)
            try:
                cur.execute("SELECT agent_name FROM agent_identities WHERE agent_key=%s", (agent_key,))
                row = cur.fetchone()
            finally:
                cur.close()
    except Exception as e:
        return sanic_json({"ok": False, "error": f"auth lookup failed: {e}"}, status=500)
    if not row:
        return sanic_json({"ok": False, "error": "unknown agent_key"}, status=403)

    try:
        fk = await asyncio.to_thread(bootstrap_client.get_fernet_key)
    except Exception as e:
        return sanic_json({"ok": False, "error": f"local fernet key load failed: {e}"}, status=500)

    return sanic_json({
        "ok": True,
        "fernet_key": fk,
        "is_active": True,
        "tier": "Community",
        "expires_at": None,
    })






@app.get("/api/agent/deploy/linux")
async def deploy_agent_linux(request):
    proto = "https" if request.scheme == "https" else "http"
    host = request.host
    token = (request.args.get("token") or "").strip()
    server_url = f"{proto}://{host}"
    server_ip = host.split(":")[0]
    script = _render_linux_install(server_url, server_ip, token)
    return HTTPResponse(body=script, content_type="text/x-sh")


@app.get("/api/agent/deploy/windows")
async def deploy_agent_windows(request):
    proto = "https" if request.scheme == "https" else "http"
    host = request.host
    token = (request.args.get("token") or "").strip()
    server_url = f"{proto}://{host}"
    server_ip = host.split(":")[0]
    script = _render_windows_install(server_url, server_ip, token)
    return HTTPResponse(body=script, content_type="text/plain")


class CustomEncoder(pyjson.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, (datetime,)):
            return obj.strftime("%Y-%m-%d %H:%M:%S")
        if isinstance(obj, float) and 1000000000 < obj < 2000000000:
            try:
                return datetime.fromtimestamp(obj).strftime("%Y-%m-%d %H:%M:%S")
            except Exception:
                pass
        
        if isinstance(obj, str) and '.' in obj:
            try:
                f_val = float(obj)
                if 1000000000 < f_val < 2500000000:
                   return datetime.fromtimestamp(f_val).strftime("%Y-%m-%d %H:%M:%S")
            except ValueError:
                pass
                
        return str(obj)

def fetch_email_config():
    try:
        with sync_mysql_conn("userdb") as conn:
            cursor = conn.cursor(dictionary=True)
            try:
                cursor.execute("SELECT * FROM email_config ORDER BY updated_at DESC LIMIT 1")
                return cursor.fetchone()
            finally:
                cursor.close()
    except Exception as e:
        print(f"[!] Email config fetch error: {e}")
        return None

def render_template(template: str, context: dict) -> str:
    for key, value in context.items():
        template = template.replace(f"{{{{{key}}}}}", str(value))
    return template

 


def get_mail_config():
    """Get mail configuration and determine if mail is enabled"""
    cfg = fetch_email_config()
    mail_enabled = bool(cfg and cfg.get('smtp_server') and cfg.get('email_to'))
    
    if mail_enabled:
        return {
            'enabled': True,
            'smtp_server': cfg['smtp_server'],
            'smtp_port': cfg['smtp_port'],
            'smtp_user': cfg['smtp_user'],
            'smtp_password': cfg['smtp_password'],
            'use_tls': cfg['smtp_use_tls'],
            'from_addr': cfg['email_from'],
            'to_addr': cfg['email_to']
        }
    else:
        return {'enabled': False}


async def get_role_permissions(role_name: str) -> list:
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        await cursor.execute("""
            SELECT p.name FROM roles r
            JOIN role_permissions rp ON rp.role_id = r.id
            JOIN permissions p ON p.id = rp.permission_id
            WHERE r.role_name = %s OR r.role_name = 'admin'
        """, (role_name,))
        rows = await cursor.fetchall()
        await cursor.close(); await cnx.close()
        perms = list(set([row[0] for row in rows]))
        return perms
    except Exception as e:
        print(f"Error fetching role permissions: {e}")
        return []

async def get_user_permissions(user_id: int) -> list:
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        await cursor.execute("""
            SELECT p.name FROM users u
            JOIN roles r ON u.role = r.role_name
            JOIN role_permissions rp ON rp.role_id = r.id
            JOIN permissions p ON p.id = rp.permission_id
            WHERE u.id = %s OR (u.role = 'admin' AND p.name = 'all_permission')
        """, (user_id,))
        rows = await cursor.fetchall()
        await cursor.close(); await cnx.close()
        perms = [row[0] for row in rows]
        if 'all_permission' in perms:
            pass
        return perms
    except Exception as e:
        print(f"Error fetching permissions: {e}")
        return []

@app.route("/user/permissions", methods=["GET"])
async def fetch_my_permissions(request):
    user_id = current_user_id(request)
    if not user_id:
        return sanic_json({"status": "error", "message": "Not authenticated"}, status=401)
    perms = await get_user_permissions(user_id)
    return sanic_json({"status": "success", "permissions": perms})

async def user_has_permission(user_id: int, permission_name: str) -> bool:
    if not user_id:
        return False
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        await cursor.execute("""
            SELECT COUNT(*) FROM users u
            JOIN roles r ON u.role = r.role_name
            JOIN role_permissions rp ON rp.role_id = r.id
            JOIN permissions p ON p.id = rp.permission_id
            WHERE u.id = %s AND (p.name = %s OR p.name = 'all_permission')
        """, (user_id, permission_name))

        result = await cursor.fetchone()
        await cursor.close(); await cnx.close()
        return result[0] > 0
    except Exception as e:
        print(f"[!] Permission check error: {e}")
        return False



async def audit_log(request, action: str, resource: str, details: str = ""):
    # Attributed from the session, not from a client-supplied header — an
    # audit trail an attacker can sign with someone else's name is worse than
    # no audit trail, because it reads as authoritative.
    user_id = current_user_id(request)
    username = current_username(request)

    ip = _client_ip(request)

    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        await cursor.execute("""
            INSERT INTO audit_logs (user_id, username, action, resource, details, ip_address)
            VALUES (%s, %s, %s, %s, %s, %s)
        """, (user_id, username, action, resource, details, ip))

        await cnx.commit()
        await cursor.close(); await cnx.close()

        try:
            _spawn(os_utils.index_log(
                agent="system",
                table="audit_logs",
                item={
                    "user_id": user_id,
                    "username": username,
                    "action": action,
                    "resource": resource,
                    "details": details,
                    "ip_address": ip
                }
            ))
        except Exception as os_e:
            print(f"[!] OpenSearch audit index failed: {os_e}")
    except Exception as e:
        print(f"[!] Audit log failure: {e}")

# ---------------------------------------------------------------------------
# Authentication / authorization
#
# Identity comes from an HttpOnly session cookie, never from a client-supplied
# header. `X-User-ID` is still sent by the frontend and still validated, but
# only as an assertion that must MATCH the session — a mismatch is a hard 401.
# That check is not redundant: browsers cannot attach custom headers to
# cross-site requests without a CORS preflight, so requiring the header also
# makes the session cookie unusable for CSRF.
# ---------------------------------------------------------------------------

# Handlers reachable without a session. Everything else is deny-by-default.
_PUBLIC_HANDLERS = {
    # Login endpoint and the SPA shell (the SPA routes to /login on its own).
    "login", "logout", "health_check", "serve_root", "serve_index",
    # Agent-facing endpoints. Public here means "no browser session", not
    # "no authentication": each one validates X-Agent-Key or an enrollment
    # token itself. The session layer must not be what gates agent traffic,
    # or every enrolled endpoint stops reporting the moment this ships.
    #
    # The automation poll/report pair really did carry nothing until now, and
    # the consequence was not a bypass but a way to blind the console: walking
    # task_id upward with {"status":"SUCCESS"} closed queued response actions
    # before the agent polled them, so they never ran and the UI showed them
    # green. They authenticate now — see _require_agent.
    "download_agent", "register_agent", "agent_bootstrap",
    # See the path list below: the pending token is the credential.
    "complete_second_factor",
    # The same login finished with a security key. Public in exactly the sense
    # `complete_second_factor` is: no browser session exists yet, and the
    # credential is the pending token minted moments earlier by a correct
    # password — single use, five minutes, able to do one thing.
    #
    # These two were added to `_PUBLIC_EXACT_PATHS` first, which is the wrong
    # list and says so in its own comment: it is a path-shaped *mirror* used
    # only as a backstop when the router cannot resolve a handler. `authenticate`
    # checks this set, matches on handler name, and returns 401 long before the
    # backstop is consulted — so both routes answered "Authentication required"
    # on the login page, in half a millisecond, without running.
    "webauthn_login_begin", "webauthn_login_finish",
    # The CA certificate, and only the certificate. It is the half of the pair
    # whose purpose is to be handed out, an installer needs it before it holds
    # any credential, and publishing it grants nothing - it lets a client check
    # a signature it could not otherwise check.
    "download_ca",
    "deploy_agent_linux", "deploy_agent_windows",
    "get_pending_automations_for_agent",
    "report_automation_result", "report_automation_result_by_id",
    # The channel every agent opens. Same sense of "public" as the rest of
    # this list and more load-bearing than any of them: it carries
    # /self_destruct, and `_validate_agent_auth_sync` resolves an identity
    # from `agent_identities` before a link is registered, refusing the
    # fleet-wide secret outright. Counting it as session-only would be a
    # false reassurance in the opposite direction - it is not gated by an
    # operator session, and it must not be.
    "agent_link_socket",
}

# The only handlers a session on a seeded password may reach. `logout` is
# included so an operator who cannot set a password can still end the session
# rather than being stuck in it.
_PASSWORD_CHANGE_HANDLERS = {"change_password", "logout", "serve_root",
                             "serve_index", "health_check"}

# Static assets must load before anyone can log in. Matched on path because
# Sanic's internal static handler name varies across versions.
_PUBLIC_PATH_PREFIXES = ("/assets/", "/vite.svg", "/favicon.ico")

# Path-shaped mirror of _PUBLIC_HANDLERS, used only as the backstop in
# `authenticate` when a route's handler cannot be resolved. Kept deliberately
# narrow: it covers login and agent traffic, not the SPA catch-all, so a
# failure here is a visibly broken UI rather than a silent bypass.
_PUBLIC_EXACT_PATHS = {
    "/login", "/logout", "/health",
    "/api/agents/register", "/api/agents/bootstrap",
    "/api/agent/deploy/linux", "/api/agent/deploy/windows",
    # Agent traffic, not public: the handler authenticates with X-Agent-Key
    # before it registers anything, and refuses the fleet-wide secret. Listed
    # here only because the operator session middleware would otherwise reject
    # the handshake before the handler is ever reached.
    "/agent-link",
    # The second half of a login. There is no session yet by definition - the
    # credential is the pending token, minted moments earlier by a correct
    # password, single-use, five-minute lifetime, and able to do exactly one
    # thing. Gating it on a session would make the second factor unreachable.
    "/login/2fa",
    # The same login, finished with a security key instead of a code, and
    # public for exactly the same reason: the pending token is the credential
    # and there is no session to require.
    #
    # These were added to the auth-wiring test's list of session-only routes
    # and not here, which reads as the same decision and is not: that list
    # says "no permission check is expected", this one says "the session
    # middleware must not reject it first". So the middleware answered 401
    # before either handler ran, and the console showed "Authentication
    # required" on a login page - where the whole point is that the operator
    # is not authenticated yet.
    "/login/2fa/webauthn/begin",
    "/login/2fa/webauthn/finish",
    # The CA *certificate*, so an agent can verify a self-signed server.
    # Public on purpose: a certificate is the half of the pair meant to be
    # handed out, and an installer needs it before it has any credential.
    "/api/agent/ca",
}
_AGENT_PATH_SUFFIXES = ("/automations/pending", "/automations/report")


def _is_public_path(path: str) -> bool:
    if path in _PUBLIC_EXACT_PATHS or path.startswith(_PUBLIC_PATH_PREFIXES):
        return True
    if path.startswith("/api/agent/download/"):
        return True
    if path.endswith(_AGENT_PATH_SUFFIXES):
        return True
    return bool(re.fullmatch(r"/automations/\d+/report", path))

# Permissions required per route, keyed by handler __qualname__. A set rather
# than a single value because four routes stack two decorators, and stacked
# decorators mean "all of these", not "the outermost one".
_PERMISSION_REQUIREMENTS: dict[str, set[str]] = {}


def require_permission(permission_name):
    """Declare the permission a route requires.

    Records the requirement in `_PERMISSION_REQUIREMENTS` *and* returns a
    guarding wrapper. Both halves matter, because the two decorator orderings
    do not behave the same way:

        @require_permission("x")     |    @app.route("/p")
        @app.route("/p")             |    @require_permission("x")
        async def h(...)             |    async def h(...)

    In the left form `app.route` registers the *bare* handler before this
    decorator ever runs, so the wrapper below is built but never called — which
    is how 85 routes in this file ended up silently unguarded. The registry is
    consulted by the `authenticate` middleware via the handler's __qualname__,
    which @wraps preserves, so enforcement is now identical either way.

    Prefer the right-hand form in new code: the wrapper is the cheaper check
    when it does run.
    """
    def decorator(func):
        _PERMISSION_REQUIREMENTS.setdefault(func.__qualname__, set()).add(permission_name)

        @wraps(func)
        async def wrapper(request, *args, **kwargs):
            # The middleware already resolved this; skip the duplicate query.
            if not getattr(request.ctx, "permission_checked", False):
                if not await user_has_permission(current_user_id(request), permission_name):
                    # One is a mis-click. A run of them from one session is
                    # somebody mapping what they can touch, and that only
                    # looks like anything once the attempts are counted
                    # together - which is what the fold in `self_defence`
                    # does.
                    await _platform_event(
                        "PERMISSION_DENIED",
                        subject=str(current_user_id(request) or "unknown"),
                        source_ip=_client_ip(request),
                        detail=f"reached for {permission_name!r} on "
                               f"{request.path}")
                    return sanic_json({
                        "status": "error",
                        "message": f"Permission denied: '{permission_name}' required"
                    }, status=403)
            return await func(request, *args, **kwargs)

        # @wraps copies __qualname__, so this is the same key — the setdefault
        # above already holds it. Restated for the case where a future change
        # makes the wrapper's qualname differ.
        _PERMISSION_REQUIREMENTS.setdefault(wrapper.__qualname__, set()).add(permission_name)
        return wrapper
    return decorator


def current_user(request) -> dict | None:
    """The authenticated user for this request, or None."""
    return getattr(request.ctx, "user", None)


def current_user_id(request) -> int:
    user = current_user(request)
    return int(user["user_id"]) if user else 0


def current_username(request) -> str:
    user = current_user(request)
    return user["username"] if user else "Anonymous"


def _unauthorized(message: str):
    return sanic_json({"status": "error", "message": message}, status=401)


async def revoke_user_sessions(user_id: int, reason: str = "") -> int:
    """Kill every live session for a user.

    Called whenever the credential or the privileges a session was issued
    against change — password reset, role change, account deletion. Without
    this an admin can revoke someone's access and still have their old browser
    tab working until the idle timeout.
    """
    if not user_id:
        return 0
    try:
        async with userdb_conn() as cnx:
            cur = await cnx.cursor()
            try:
                killed = await session_store.revoke_for_user(cur, int(user_id))
                await cnx.commit()
            finally:
                await cur.close()
        if killed:
            print(f"[Session] Revoked {killed} session(s) for user {user_id} ({reason})")
        return killed
    except Exception as e:
        print(f"[!] Session revoke failed for user {user_id}: {e}")
        return 0


@app.on_response
async def _release_request_db_connections(request, response):
    """Return every pooled connection this request acquired.

    Runs on error responses too, which is the whole point: the leak was on the
    exception path, where handlers returned a 500 without reaching their
    close. Sanic runs response middleware for those the same way.

    `_PooledConn.close()` is idempotent, so handlers that already released
    theirs cost nothing here. Failures are swallowed per connection: a
    connection we cannot return must not turn a successful response into an
    error, and the pool's idle-reaper will discard it.
    """
    held = getattr(request.ctx, "_db_conns", None)
    if not held:
        return
    request.ctx._db_conns = []
    for pooled in held:
        try:
            await pooled.close()
        except Exception as e:
            print(f"[!] releasing a pooled connection failed: {e}", flush=True)


_SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})


@app.on_request
async def authenticate(request):
    """Resolve the session cookie, then gate the request.

    Runs after routing, so `request.route` already names the handler we are
    about to call. Returning a response here short-circuits that handler.
    """
    request.ctx.user = None
    request.ctx.session_token = None
    request.ctx.permission_checked = False

    # CORS preflight carries no cookies by design.
    if request.method == "OPTIONS":
        return None

    if request.path.startswith(_PUBLIC_PATH_PREFIXES):
        return None

    # Resolved before the public-route check on purpose: `download_agent` is
    # public at this layer but still reads the session to decide whether the
    # caller is an operator or an enrolled agent.
    raw_token = request.cookies.get(session_store.SESSION_COOKIE)
    if raw_token:
        try:
            async with userdb_conn() as cnx:
                cur = await cnx.cursor(dictionary=True)
                try:
                    row = await session_store.load(cur, raw_token)
                    if row:
                        request.ctx.user = row
                        request.ctx.session_token = raw_token
                        if await session_store.touch(cur, row["token_hash"], row["last_seen_at"]):
                            await cnx.commit()
                finally:
                    await cur.close()
        except Exception as e:
            print(f"[!] Session lookup failed: {e}")

    user = request.ctx.user

    route = getattr(request, "route", None)
    handler = getattr(route, "handler", None) if route else None
    handler_name = getattr(handler, "__qualname__", "")

    # Public routes are exempt from everything below. This has to come before
    # the X-User-ID check: the browser keeps `userId` in localStorage and the
    # axios interceptor attaches it to /login too, so enforcing the header
    # first would make an expired session impossible to log back in from.
    if handler_name in _PUBLIC_HANDLERS:
        return None

    # Backstop for the case where the router did not attach a resolved route to
    # the request. Everything below is deny-by-default, so without this a Sanic
    # release that changes when `request.route` is populated would lock
    # operators out of the login page rather than let anything through.
    if not handler_name and _is_public_path(request.path):
        return None

    if not user:
        return _unauthorized("Authentication required")

    # A session still on the seeded password may do exactly one thing.
    #
    # db/init_userdb.sql ships `admin` with a bcrypt hash that is published in
    # this repository, and nothing ever required it to be changed. A console
    # that isolates endpoints and runs commands on them was reachable with a
    # credential anyone could read.
    #
    # Enforced here rather than in the UI: a flag the front end is trusted to
    # honour is not a control, it is a suggestion. `logout` stays open so
    # somebody who cannot set a password is not stuck in the session.
    if user.get("must_change_password") and handler_name not in _PASSWORD_CHANGE_HANDLERS:
        return sanic_json({
            "status": "error",
            "code": "must_change_password",
            "message": "Set a new password before using the console.",
        }, status=403)

    # `X-User-ID` is no longer identity — it is an assertion checked against
    # the session. SameSite is the primary CSRF control; requiring the header
    # on state-changing requests is the second layer, and the one that still
    # holds if SameSite is ever relaxed to None for a split-origin deployment.
    asserted = request.headers.get("X-User-ID")
    if asserted is None:
        if request.method not in _SAFE_METHODS:
            return _unauthorized("X-User-ID header required on state-changing requests")
    else:
        try:
            asserted_id = int(asserted)
        except (TypeError, ValueError):
            return _unauthorized("Malformed X-User-ID header")
        if asserted_id != int(user["user_id"]):
            return _unauthorized("X-User-ID does not match the active session")

    required = _PERMISSION_REQUIREMENTS.get(handler_name)
    if required:
        uid = int(user["user_id"])
        for permission in sorted(required):
            if not await user_has_permission(uid, permission):
                return sanic_json({
                    "status": "error",
                    "message": f"Permission denied: '{permission}' required"
                }, status=403)
        request.ctx.permission_checked = True

    return None


async def _record_agent_version(agent: str, reported: str | None) -> None:
    """Remember what this agent said it was, on every connect.

    On connect rather than on enrolment, because the thing worth knowing is
    what is *running* - an agent enrolled a year ago and upgraded twice since
    would otherwise be recorded as whatever it was the day it first appeared.

    NULL is a real answer here and not a gap: it means the agent sent no
    version header at all, so it predates version reporting. That is a
    different fact from "old but known", and the console needs to tell them
    apart - one is a fleet member to upgrade, the other is one nobody can even
    name.

    Failing to write this must never fail the handshake. A version is a
    diagnosis; losing it costs a line in the console, and refusing the channel
    over it would cost the endpoint.
    """
    hyphen, underscore, _ = _agent_name_forms(agent)
    candidates = [n for n in dict.fromkeys([agent, hyphen, underscore]) if n]
    try:
        cnx = await connect_userdb()
        cur = await cnx.cursor()
        try:
            placeholders = ",".join(["%s"] * len(candidates))
            await cur.execute(
                f"UPDATE agent_identities SET agent_version = %s "
                f"WHERE agent_name IN ({placeholders})",
                [reported] + candidates,
            )
            await cnx.commit()
        finally:
            await cur.close()
            await cnx.close()
    except Exception as e:
        print(f"[agent-link] could not record {agent}'s version: {e}", flush=True)


def _handler_qualname(handler) -> str:
    """The name a route's handler was registered under, through any wrapping.

    Sanic wraps a websocket handler in a `functools.partial`, which has no
    `__qualname__` - so every websocket route landed in the "unresolvable"
    bucket and the boot-time audit simply did not cover them:

        [Auth] WARNING: 3 route(s) with an unresolvable handler:
        ['/agent-link', '/console-proxy/<agent:str>', '/vnc-proxy/<agent:str>']

    Those three are the agent channel, a root shell on an endpoint, and a live
    view of its screen. The one category the audit could not see was the one
    worth seeing, and it said so in a line that reads like a formatting
    complaint.
    """
    seen = set()
    while handler is not None and id(handler) not in seen:
        seen.add(id(handler))
        name = getattr(handler, "__qualname__", "")
        if name:
            return name
        handler = getattr(handler, "func", None) or getattr(handler, "__wrapped__", None)
    return ""


@app.before_server_start
async def verify_auth_wiring(app):
    """Report the auth posture of every route at boot.

    The bug this replaces was invisible precisely because nothing ever
    inspected the result of wiring `@require_permission` up: 85 routes looked
    guarded in the source and were not. Printing the tally at startup means a
    regression shows up in the very first lines of the container log.
    """
    try:
        guarded = public = session_only = 0
        unknown: list[str] = []

        for route in app.router.routes:
            name = _handler_qualname(getattr(route, "handler", None))
            if not name:
                unknown.append("/" + "/".join(getattr(route, "parts", ())))
            elif name in _PUBLIC_HANDLERS:
                public += 1
            elif name in _PERMISSION_REQUIREMENTS:
                guarded += 1
            else:
                session_only += 1

        print(
            f"[Auth] Routes: {guarded} permission-gated, {session_only} session-only, "
            f"{public} public."
        )
        if not _PERMISSION_REQUIREMENTS:
            print("[Auth] FATAL: permission registry is empty — every route is session-only.")
        if unknown:
            print(f"[Auth] WARNING: {len(unknown)} route(s) with an unresolvable handler: {unknown[:5]}")
    except Exception as e:
        # Diagnostics must never be what stops the server from booting.
        print(f"[Auth] Wiring self-check skipped: {e}")

DEFAULT_ALERT_TEMPLATE = "Critical Alerts (default)"


def send_email(template_name: str, context: dict, fallback: str | None = None) -> bool:
    """Send a templated mail.

    `dispatch_critical_alerts` names its template per agent
    ("Critical Alerts - Agent: WIN-01"), so an operator would have to hand-create
    one row for every enrolled endpoint before any alert mail went out — and the
    only symptom of not having done so was a line in the server log. `fallback`
    keeps the per-agent override working while letting a generic template cover
    the agents nobody has customised.
    """
    config = get_mail_config()
    if not config['enabled']:
        print("[!] Email not enabled or configured")
        return False

    try:
        with sync_mysql_conn("userdb") as conn:
            cursor = conn.cursor(dictionary=True)
            try:
                cursor.execute("SELECT * FROM email_templates WHERE template_name = %s", (template_name,))
                template = cursor.fetchone()
                if not template and fallback:
                    cursor.execute("SELECT * FROM email_templates WHERE template_name = %s", (fallback,))
                    template = cursor.fetchone()
                    if template:
                        print(f"[i] Template '{template_name}' not found; using '{fallback}'.")
            finally:
                cursor.close()

        if not template:
            print(f"[!] Template '{template_name}' not found"
                  + (f" (fallback '{fallback}' missing too)." if fallback else "."))
            return False

        subject = render_template(template['subject_template'], context)
        body = render_template(template['body_template'], context)

        msg = EmailMessage()
        msg['Subject'] = subject
        msg['From'] = config['from_addr']
        msg['To'] = config['to_addr']
        msg.set_content(body)

        with smtplib.SMTP(config['smtp_server'], config['smtp_port']) as server:
            if config['use_tls']:
                server.starttls()
            if config['smtp_user'] and config['smtp_password']:
                server.login(config['smtp_user'], config['smtp_password'])
            server.send_message(msg)

        print(f"[+] Templated mail sent: {subject}")
        return True

    except Exception as e:
        print(f"[!] Templated email send error: {e}")
        return False

def dispatch_critical_alerts(agent: str, limit: int = 100) -> int:
    db_name = _agent_db_name(agent)
    try:
        with sync_mysql_conn(db_name) as conn:
            cursor = conn.cursor(dictionary=True)
            try:
                cursor.execute("""
                    SELECT * FROM events_alert
                    WHERE (sent IS NULL OR sent = FALSE)
                    ORDER BY id DESC LIMIT %s
                """, (limit,))
                rows = cursor.fetchall()

                lines: List[str] = []
                ids: List[int] = []

                for r in rows:
                    try:
                        severity = str(r.get('severity', '')).strip().upper()
                        if severity == 'CRITICAL':
                            line = f"[{r.get('timestamp','')}] Agent: {agent} | Source: {r.get('source','')} | Categories: {r.get('categories','')} | Message: {r.get('message','')}"
                            lines.append(line)
                            ids.append(r['id'])
                    except Exception as parse_err:
                        print(f"[!] Parse error for agent {agent}, alert ID {r.get('id')}: {parse_err}")

                if lines:
                    body = f"Critical Alerts from Agent: {agent}\n\n" + '\n'.join(lines)
                    send_email(f'Critical Alerts - Agent: {agent}', {"body": body, "agent": agent},
                               fallback=DEFAULT_ALERT_TEMPLATE)
                    cursor.executemany("UPDATE events_alert SET sent = TRUE WHERE id = %s", [(i,) for i in ids])
                    conn.commit()
                    print(f"[+] Processed {len(lines)} critical alerts for agent {agent}")

                return len(lines)
            finally:
                cursor.close()

    except Exception as db_err:
        print(f"[!] dispatch_critical_alerts DB error for agent {agent}: {db_err}")
        return 0

async def load_ai_config(agent: str):
    """Load AI configuration. Currently global for the server/agent."""
    default_config = {
        "provider": "ollama",
        "api_url": "http://host.docker.internal:11434/api",
        "model": "llama3:latest",
        "api_key": ""
    }
    try:
        async with userdb_conn() as cnx:
            cur = await cnx.cursor(dictionary=True)
            try:
                await cur.execute("SELECT * FROM ai_config ORDER BY updated_at DESC LIMIT 1")
                row = await cur.fetchone()
            finally:
                await cur.close()
        return row if row else default_config
    except Exception as e:
        print(f"[!] AI config fetch error: {e}, falling back to defaults")
        return default_config

def _fetch_rows_safely(db_name: str, query: str, params: tuple = (), context: str = "") -> list:
    try:
        with sync_mysql_conn(db_name) as conn:
            cursor = conn.cursor(dictionary=True)
            try:
                cursor.execute(query, params)
                return cursor.fetchall()
            finally:
                cursor.close()
    except Exception as e:
        print(f"[!] Error fetching {context}: {e}")
        return []

def fetch_unsent_siem_logs(agent: str, limit: int = 100):
    return _fetch_rows_safely(
        _agent_db_name(agent),
        "SELECT * FROM siem_events WHERE (ai_analyzed IS NULL OR ai_analyzed = FALSE) ORDER BY id ASC LIMIT %s",
        (limit,),
        f"unsent SIEM logs for {agent}",
    )

def fetch_critical_files_data(agent: str, limit: int = 100):
    return _fetch_rows_safely(
        _agent_db_name(agent),
        "SELECT * FROM critical_files ORDER BY id DESC LIMIT %s",
        (limit,),
        f"critical files for {agent}",
    )

def fetch_events_alert_data(agent: str, limit: int = 100):
    return _fetch_rows_safely(
        _agent_db_name(agent),
        "SELECT * FROM events_alert ORDER BY id DESC LIMIT %s",
        (limit,),
        f"events alert for {agent}",
    )

def fetch_vulnerabilities_data(agent: str, limit: int = 100):
    return _fetch_rows_safely(
        _agent_db_name(agent),
        "SELECT * FROM vulnerabilities_report ORDER BY id DESC LIMIT %s",
        (limit,),
        f"vulnerabilities_report for {agent}",
    )

def fetch_soar_actions_data(agent: str, limit: int = 100):
    return _fetch_rows_safely(
        _agent_db_name(agent),
        "SELECT * FROM soar_actions ORDER BY id DESC LIMIT %s",
        (limit,),
        f"soar_actions for {agent}",
    )

from core import mq as mq_utils

async def analyze_agent_logs(agent: str, limit: int = 100):
    """
    Refactored: Fetch logs and push to ai_manual_queue in chunks (background)
    to avoid timeout for the HTTP request.
    """
    cfg = await load_ai_config(agent)
    if not cfg:
        return {'success': False, 'error': f'AI config missing for {agent}', 'logs_analyzed': 0}

    try:
        key_b64 = getattr(app.ctx, "fernet_key", None)
        if not key_b64:
            key_b64 = await asyncio.to_thread(bootstrap_client.get_fernet_key)
        fernet_obj = Fernet(key_b64.encode("utf-8") if isinstance(key_b64, str) else key_b64)
    except Exception as e:
        return {'success': False, 'error': f'Fernet key error: {e}'}

    siem_rows   = await asyncio.to_thread(fetch_unsent_siem_logs, agent, limit)
    cf_rows     = await asyncio.to_thread(fetch_critical_files_data, agent, limit)
    alert_rows  = await asyncio.to_thread(fetch_events_alert_data, agent, limit)
    vuln_rows   = await asyncio.to_thread(fetch_vulnerabilities_data, agent, limit)
    soar_rows   = await asyncio.to_thread(fetch_soar_actions_data, agent, limit)

    all_logs = []
    for r in siem_rows:  all_logs.append(("siem_events", r))
    for r in cf_rows:    all_logs.append(("critical_files", r))
    for r in alert_rows: all_logs.append(("events_alert", r))
    for r in vuln_rows:  all_logs.append(("vulnerabilities_report", r))
    for r in soar_rows:  all_logs.append(("soar_actions", r))

    if not all_logs:
        return {'success': True, 'message': 'No new logs to analyze.', 'logs_analyzed': 0}

    chunk_size = 10
    pushed_count = 0
    msgs_published = 0

    by_table: dict = {}
    for table, row in all_logs:
        dec = decrypt_row_fields(row, ENCRYPTED_FIELDS_MAP.get(table, []), fernet_obj)
        by_table.setdefault(table, []).append(dec)

    for table, rows in by_table.items():
        for i in range(0, len(rows), chunk_size):
            batch = rows[i:i + chunk_size]
            await mq_utils.publish_to_queue(mq_utils.AI_MANUAL, agent, table, batch)
            pushed_count += len(batch)
            msgs_published += 1

    return {
        'success': True,
        'message': f"Queued {pushed_count} logs across {msgs_published} batches for background analysis.",
        'logs_queued': pushed_count,
        'batches_published': msgs_published,
        'total_records': len(all_logs)
    }

async def analyze_selected_logs(agent: str, logs: list):
    """Analyze a specific selection of logs from the UI"""
    try:
        key_b64 = getattr(app.ctx, "fernet_key", None)
        if not key_b64:
            key_b64 = await asyncio.to_thread(bootstrap_client.get_fernet_key)
        fernet_obj = Fernet(key_b64.encode("utf-8") if isinstance(key_b64, str) else key_b64)
    except Exception as e:
        return {'success': False, 'error': f'Fernet key error: {e}'}

    pushed_count = 0
    for log in logs:
        table = log.get("table")
        data  = log.get("data")
        if table and data:
            log_id = data.get("id")
            if log_id:
                row = await asyncio.to_thread(fetch_one_log_from_db, agent, table, log_id)
                if row:
                    dec = decrypt_row_fields(row, ENCRYPTED_FIELDS_MAP.get(table, []), fernet_obj)
                    await mq_utils.publish_to_queue(mq_utils.AI_MANUAL, agent, table, dec)
                    pushed_count += 1
    
    return {'success': True, 'message': f"Queued {pushed_count} selected logs for analysis."}

def fetch_one_log_from_db(agent, table, log_id):
    """Helper to fetch a single log row for re-decryption"""
    db_name = _agent_db_name(agent)
    try:
        with sync_mysql_conn(db_name) as conn:
            cursor = conn.cursor(dictionary=True)
            try:
                cursor.execute(f"SELECT * FROM {_quote_identifier(table)} WHERE id = %s", (log_id,))
                return cursor.fetchone()
            finally:
                cursor.close()
    except Exception:
        return None


async def analyze_all_agents_logs():
    try:
        def get_agents():
            with sync_mysql_conn() as conn:
                cursor = conn.cursor()
                try:
                    cursor.execute("SHOW DATABASES")
                    databases = cursor.fetchall()
                    return [db[0].replace('_db', '') for db in databases if db[0].endswith('_db')]
                finally:
                    cursor.close()

        agents = await asyncio.to_thread(get_agents)
        
        tasks = [analyze_agent_logs(agent) for agent in agents]
        agent_results = await asyncio.gather(*tasks, return_exceptions=True)
        
        results = {}
        total_analyzed = 0
        
        for agent, result in zip(agents, agent_results):
            if isinstance(result, Exception):
                results[agent] = {"success": False, "error": str(result), "logs_analyzed": 0}
            else:
                results[agent] = result
                total_analyzed += result.get('logs_analyzed', 0)
        
        return {
            'success': True,
            'total_logs_analyzed': total_analyzed,
            'agents_processed': len(agents),
            'results': results
        }
        
    except Exception as e:
        return {
            'success': False,
            'error': f'Error analyzing logs for all agents: {str(e)}',
            'total_logs_analyzed': 0
        }

@app.route("/api/global/stats", methods=["GET"])
@require_permission("read_telemetry")
async def get_global_inventory_stats(request):
    try:
        def fetch_counts():
            """One query instead of three per agent.

            This looped `USE <db>` then three COUNT(*)s per agent on a single
            connection — 3N round trips, serial. information_schema keeps
            approximate row counts for InnoDB and answers for the whole fleet
            at once. "Approximate" is fine here: these are headline inventory
            totals, not anything a decision hangs on.
            """
            with sync_mysql_conn() as conn:
                cursor = conn.cursor()
                try:
                    cursor.execute("""
                        SELECT TABLE_NAME, SUM(TABLE_ROWS)
                          FROM information_schema.TABLES
                         WHERE TABLE_SCHEMA LIKE %s
                           AND TABLE_NAME IN ('hardware_inventory','software_inventory','fim_data')
                         GROUP BY TABLE_NAME
                    """, ('%\\_db',))
                    counts = {name: int(n or 0) for name, n in cursor.fetchall()}
                    return (
                        counts.get('hardware_inventory', 0),
                        counts.get('software_inventory', 0),
                        counts.get('fim_data', 0),
                    )
                finally:
                    cursor.close()

        hw, sw, fim = await asyncio.to_thread(fetch_counts)
        return sanic_json({
            "status": "success",
            "total_hardware": hw,
            "total_software": sw,
            "total_fim_events": fim
        })
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

def dispatch_all_critical_alerts():
    """Dispatch critical alerts for all agents"""
    try:
        with sync_mysql_conn() as conn:
            cursor = conn.cursor()
            try:
                cursor.execute("SHOW DATABASES")
                databases = cursor.fetchall()
                agents = [db[0].replace('_db', '') for db in databases if db[0].endswith('_db')]
            finally:
                cursor.close()

        total_alerts = 0
        for agent in agents:
            total_alerts += dispatch_critical_alerts(agent)

        print(f"[+] Total critical alerts processed: {total_alerts}")
        return total_alerts

    except Exception as e:
        print(f"[!] Error in dispatch_all_critical_alerts: {e}")
        return 0

def _close_sync_quiet(conn):
    try:
        if conn is not None:
            conn.close()
    except Exception:
        pass

async def _close_async_quiet(conn):
    """Return a connection, whatever kind it is, without ever raising.

    This used to read

        await conn.ensure_closed() if hasattr(conn, "ensure_closed") else conn.close()

    and `await` binds tighter than a conditional expression, so Python parses
    that as `(await conn.ensure_closed()) if ... else conn.close()`. The else
    branch was never awaited - and `_PooledConn.close` is a coroutine, so the
    connection went back to the pool only when the garbage collector got round
    to `__del__`, with

        RuntimeWarning: coroutine '_PooledConn.close' was never awaited

    the only sign. Not a leak, because `__del__` catches it, but a pool that
    hands back its slots on GC timing is a pool that runs out under exactly
    the load that makes GC late.

    `close()` first, deliberately. On `_PooledConn` that is the method that
    releases the slot; `ensure_closed` would reach through `__getattr__` to the
    underlying connection and shut the socket without ever telling the pool -
    turning a warning into the slot leak it was only pretending to be.
    """
    if conn is None:
        return
    try:
        result = conn.close()
        if inspect.isawaitable(result):
            await result
        elif hasattr(conn, "ensure_closed"):
            # aiomysql: `close()` is synchronous and only marks it; this is
            # the one that actually waits for the socket.
            await conn.ensure_closed()
    except Exception:
        pass

from contextlib import contextmanager, asynccontextmanager

@contextmanager
def sync_mysql_conn(db: str | None = None):
    """Sync MySQL connection with guaranteed close on exception."""
    conn = mysql.connector.connect(
        host=DB_HOST, port=DB_PORT, user=DB_USER, password=DB_PASSWORD,
        database=db
    )
    try:
        yield conn
    finally:
        _close_sync_quiet(conn)

@asynccontextmanager
async def userdb_conn():
    cnx = await connect_userdb()
    try:
        yield cnx
    finally:
        await _close_async_quiet(cnx)

@asynccontextmanager
async def agent_conn(agent: str):
    cnx = await connect_db_for_agent(agent)
    try:
        yield cnx
    finally:
        await _close_async_quiet(cnx)


import collections

_POOL_MAXSIZE      = 10
_POOL_IDLE_SEC     = 60
# Shorter than Sanic's response timeout, so an exhausted pool surfaces as its
# own error rather than as a generic request timeout.
_POOL_ACQUIRE_TIMEOUT = float(os.getenv("DB_POOL_ACQUIRE_TIMEOUT", "5"))


class _PooledConn:
    """Thin proxy over a mysql.connector.aio Connection. `await close()` returns
    the underlying connection to the owning pool instead of closing the socket."""
    __slots__ = ("_conn", "_pool", "_released")

    def __init__(self, conn, pool):
        self._conn = conn
        self._pool = pool
        self._released = False

    async def close(self):
        if self._released:
            return
        self._released = True
        await self._pool.release(self._conn)

    def __getattr__(self, name):
        return getattr(self._conn, name)

    def __del__(self):
        if self._released:
            return
        try:
            self._pool._idle.append((self._conn, time.time()))
            self._pool._sem.release()
            self._released = True
        except Exception:
            pass



def _track_for_request(pooled):
    """Attach a pooled connection to the request that acquired it.

    74 handlers acquire a connection and release it only on the happy path, so
    any exception in between leaked one permanently. The pool holds ten. Once
    ten had leaked, every endpoint in the server stopped answering.

    Fixing that by rewriting 74 handlers to use `async with` is a large diff
    across code paths with no tests, and it only holds until someone writes
    the seventy-fifth. Tying the connection to the request instead makes the
    leak structurally impossible for anything served over HTTP, whatever style
    the handler is written in: whoever acquires it, the response middleware
    releases it.

    `close()` is idempotent, so a handler that does release its own connection
    correctly is unaffected.

    Background tasks have no current request and are not covered - they still
    have to release their own. That is a much smaller surface, and
    `agent_conn` / `userdb_conn` exist for it.
    """
    try:
        request = Request.get_current()
    except Exception:
        return pooled            # no request context: a background task
    if request is None:
        return pooled
    held = getattr(request.ctx, "_db_conns", None)
    if held is None:
        held = []
        request.ctx._db_conns = held
    held.append(pooled)
    return pooled


class _AsyncMySQLPool:
    def __init__(self, factory, maxsize=_POOL_MAXSIZE, max_idle_sec=_POOL_IDLE_SEC):
        self._factory = factory
        self._sem = asyncio.Semaphore(maxsize)
        self._idle = collections.deque()
        self._lock = asyncio.Lock()
        self._max_idle_sec = max_idle_sec

    async def acquire(self):
        # Bounded. An unbounded `await self._sem.acquire()` turns a leaked
        # connection into a hang: once ten handlers had failed to release,
        # every subsequent request waited forever and Sanic returned 500 at
        # its own response timeout, ten seconds later, with a message that
        # said nothing about the pool. Agents polling `automations/pending`
        # then saw nothing but 500s.
        #
        # Failing fast does not fix the leak, but it names it.
        try:
            await asyncio.wait_for(self._sem.acquire(), timeout=_POOL_ACQUIRE_TIMEOUT)
        except asyncio.TimeoutError:
            raise RuntimeError(
                f"No database connection available after "
                f"{_POOL_ACQUIRE_TIMEOUT}s (pool size {_POOL_MAXSIZE}, "
                f"{len(self._idle)} idle). A handler is acquiring connections "
                f"without releasing them on its error path."
            ) from None
        try:
            now = time.time()
            async with self._lock:
                while self._idle:
                    conn, ts = self._idle.popleft()
                    if now - ts >= self._max_idle_sec:
                        try:
                            await conn.close()
                        except Exception:
                            pass
                        continue
                    try:
                        await conn.rollback()
                    except Exception:
                        try:
                            await conn.close()
                        except Exception:
                            pass
                        continue
                    return _track_for_request(_PooledConn(conn, self))
            conn = await self._factory()
            return _track_for_request(_PooledConn(conn, self))
        except BaseException:
            self._sem.release()
            raise

    async def release(self, conn):
        try:
            await conn.rollback()
        except Exception:
            pass
        try:
            async with self._lock:
                self._idle.append((conn, time.time()))
        finally:
            self._sem.release()


_userdb_pool: "_AsyncMySQLPool | None" = None
_agent_pools: "dict[str, _AsyncMySQLPool]" = {}
_agent_pools_lock = asyncio.Lock()


def _make_userdb_factory():
    async def _f():
        return await connect(
            host=DB_HOST, port=DB_PORT, user=DB_USER,
            password=DB_PASSWORD, database="userdb",
        )
    return _f


def _agent_db_name(agent: str) -> str:
    """Map an agent identifier (possibly a Windows hostname containing '-')
    to its MySQL database name. MUST mirror server.py:_sanitize_db_name so
    every code path lands on the same DB that the ingest layer created.
    """
    safe = re.sub(r'[^A-Za-z0-9_]', '_', agent or 'agent')
    safe = safe.strip('_') or 'agent'
    return f"{safe}_db"


def _make_agent_factory(agent: str):
    db_name = _agent_db_name(agent)
    async def _f():
        try:
            return await connect(
                host=DB_HOST, port=DB_PORT, user=DB_USER,
                password=DB_PASSWORD, database=db_name,
            )
        except Exception:
            if agent.startswith("Agent_"):
                alt_db = f"{agent[6:]}_db"
                return await connect(
                    host=DB_HOST, port=DB_PORT, user=DB_USER,
                    password=DB_PASSWORD, database=alt_db,
                )
            raise
    return _f


async def connect_db_for_agent(agent: str):
    global _agent_pools
    pool = _agent_pools.get(agent)
    if pool is None:
        async with _agent_pools_lock:
            pool = _agent_pools.get(agent)
            if pool is None:
                pool = _AsyncMySQLPool(_make_agent_factory(agent))
                _agent_pools[agent] = pool
    return await pool.acquire()


async def connect_userdb():
    global _userdb_pool
    if _userdb_pool is None:
        _userdb_pool = _AsyncMySQLPool(_make_userdb_factory())
    return await _userdb_pool.acquire()

_FERNET_KEY_PATH = os.getenv(
    "FERNET_KEY_PATH",
    os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "fernet.key"),
)


def load_or_create_fernet_key() -> str:
    """Read the persisted Fernet key, or generate one on first boot."""
    path = _FERNET_KEY_PATH
    try:
        os.makedirs(os.path.dirname(path), exist_ok=True)
    except OSError as e:
        # Not fatal on its own - the directory may already exist and be
        # writable - but if it is not, the key cannot be persisted and every
        # agent bootstrapped afterwards encrypts with a key this server will
        # not have after a restart. That surfaces as undecryptable telemetry
        # long after the cause.
        print(f"[!] Could not create {os.path.dirname(path)}: {e}", flush=True)
    if os.path.exists(path):
        try:
            with open(path, "rb") as f:
                key = f.read().strip()
            if key:
                return key.decode("utf-8") if isinstance(key, bytes) else key
        except Exception as e:
            print(f"[!] Could not read existing Fernet key at {path}: {e}", flush=True)
    key = Fernet.generate_key()
    try:
        with open(path, "wb") as f:
            f.write(key)
        # This is the key every agent's telemetry is encrypted with. If the
        # mode does not take, ensure_permissions says so rather than leaving
        # a world-readable key and a cheerful success line.
        ensure_permissions(pathlib.Path(path))
        print(f"[+] Generated new Fernet key at {path}", flush=True)
    except Exception as e:
        print(f"[!] Could not persist generated Fernet key: {e}", flush=True)
    return key.decode("utf-8")


class BootstrapError(Exception):
    """Kept for legacy `except BootstrapError` blocks; never raised in CE."""


class _NoOpBootstrapClient:
    """Drop-in replacement for legacy bootstrap clients.

    Community edition: every method is a local-only no-op that returns
    "active community" status. Same shape so legacy callers keep working
    without surgery.
    """

    def __init__(self):
        self._fernet_key = None
        self.cache = {
            "active": True,
            "tier": "Community",
            "tiers": ["Community"],
            "expires_at": None,
            "fernet_key": None,
        }

    def status(self, reveal_key: bool = False) -> dict:
        if reveal_key and not self.cache.get("fernet_key"):
            self.get_fernet_key()
        return dict(self.cache)

    def get_fernet_key(self) -> str:
        if self._fernet_key is None:
            self._fernet_key = load_or_create_fernet_key()
            self.cache["fernet_key"] = self._fernet_key
        return self._fernet_key

    def validate_or_exit(self):
        self.get_fernet_key()
        print("[+] Sentora Community Edition — local key initialised.")


bootstrap_client = _NoOpBootstrapClient()


DB_POOL_WAIT_S = 60


@app.before_server_start
async def setup_db_pool(app):
    """Build the connection pool, waiting for the database to come up.

    This used to try exactly once. Under compose the app and the database
    start together and `db` is routinely not accepting connections yet, so
    `create_pool` raised, `app.ctx.db_pool` was never set - and the worker
    carried on serving requests.

    What that produced is worse than a crash, because password checks go
    through the synchronous connector and kept working. `/login` printed

        [+] Local login successful for user: admin

    then failed issuing the session, was caught by the handler's own broad
    `except`, fell through to the LDAP path, and answered 401. An operator
    with the correct password was told it was wrong, by a server reporting
    itself healthy, with the reason written only into a database table.

    So: wait, and if the database is genuinely not there, refuse to start. A
    platform whose entire job is storing telemetry has nothing useful to do
    without one, and failing to start is the only failure mode an operator
    can actually see.
    """
    print(f"[*] Initializing database connection pool to {DB_HOST}...")
    deadline = time.monotonic() + DB_POOL_WAIT_S
    delay = 1.0
    last_error = None
    while True:
        try:
            app.ctx.db_pool = await aiomysql.create_pool(
                host=DB_HOST,
                port=DB_PORT,
                user=DB_USER,
                password=DB_PASSWORD,
                db='userdb',
                autocommit=True,
                minsize=5,
                maxsize=20
            )
            print("[*] Database connection pool ready.")
            return
        except Exception as e:
            last_error = e
            if time.monotonic() >= deadline:
                break
            print(f"[*] Database not ready ({e}); retrying in {delay:.0f}s", flush=True)
            await asyncio.sleep(delay)
            delay = min(delay * 2, 8.0)

    print(f"[!] Could not reach the database at {DB_HOST}:{DB_PORT} after "
          f"{DB_POOL_WAIT_S}s: {last_error}", flush=True)
    raise RuntimeError(
        f"database unreachable at {DB_HOST}:{DB_PORT} after {DB_POOL_WAIT_S}s "
        f"({last_error}). Refusing to start: serving without a pool means "
        f"logins fail as 'invalid password' and agents report into nothing."
    )

@app.exception(Exception)
async def handle_exception(request, exception):
    if hasattr(exception, "status_code") and exception.status_code in (404, 405):
        return None

    status_code = getattr(exception, "status_code", 500)
    print(f"[!] Unhandled Exception: {exception}")
    return sanic_json({
        "status": "error",
        "message": "An internal server error occurred.",
        "details": str(exception) if app.debug else "Contact administrator"
    }, status=status_code)

@app.before_server_start
async def init_local_keys(app):
    await asyncio.to_thread(bootstrap_client.validate_or_exit)
    app.ctx.fernet_key = bootstrap_client.get_fernet_key()


@app.after_server_stop
async def close_db_pool(app):
    if hasattr(app.ctx, "db_pool"):
        print("[*] Closing database connection pool...")
        app.ctx.db_pool.close()
        await app.ctx.db_pool.wait_closed()

# Decryption failures are rate-limited per field so a whole table of
# undecryptable rows produces one line, not ten thousand.
_decrypt_failures_seen: set = set()


def decrypt_row_fields(row: dict, encrypted_fields: list, fernet: Fernet) -> dict:
    out = dict(row)
    for field in encrypted_fields or []:
        if field not in out:
            continue
        val = out[field]
        if not isinstance(val, str):
            continue
        if not val.startswith(ENC_PREFIX):
            continue
        token_b64 = val[len(ENC_PREFIX):]
        try:
            pt = fernet.decrypt(token_b64.encode("utf-8"))
            try:
                decoded = pyjson.loads(pt.decode("utf-8"))
            except Exception:
                decoded = pt.decode("utf-8")
            out[field] = decoded
            _decrypt_counters["decrypted"] += 1
        except InvalidToken:
            # Returning the raw ciphertext made a key mismatch look exactly
            # like a field nobody decrypted — the operator sees `enc::gAAA...`
            # either way, with nothing in the log to tell them which. That is
            # worth a loud line: it usually means the agent was enrolled
            # against a different Fernet key than the one this server holds,
            # which no amount of UI work will fix.
            if field not in _decrypt_failures_seen:
                _decrypt_failures_seen.add(field)
                print(
                    f"[!] Decryption failed for `{field}` — the stored value was "
                    f"encrypted with a different Fernet key than this server holds. "
                    f"Check FERNET_KEY and the agent's bootstrap; re-enrol the agent "
                    f"if it was registered against an older key.",
                    flush=True,
                )
            _decrypt_counters["undecryptable"] += 1
            out[field] = "<decryption failed — key mismatch>"
    return out


#: How many stored values this process has read, and how many it could not.
#:
#: Counted per field, and counted at all, because the scale of a key mismatch
#: was invisible. One log line per field name was the whole signal - a `set`
#: of names, printed once each - so a table where 86% of the messages could
#: not be read produced exactly the same output as one bad row. Measured on a
#: live host it was 173 of 200 in `siem_events`, 197 of 200 in `events_alert`.
#:
#: Process-local and reset on restart, deliberately. This is a health number,
#: not a ledger: "how much of what I am serving right now is unreadable".
_decrypt_counters = {"decrypted": 0, "undecryptable": 0}


def decryption_health() -> dict:
    """What proportion of encrypted values this process could actually read.

    Zero of zero is not a problem, and has to say so rather than dividing.
    """
    ok = _decrypt_counters["decrypted"]
    bad = _decrypt_counters["undecryptable"]
    total = ok + bad
    return {
        "decrypted": ok,
        "undecryptable": bad,
        "unreadable_percent": round(bad * 100.0 / total, 1) if total else 0.0,
        "detail": (
            "Values encrypted with a Fernet key this server no longer holds. "
            "There is no rotation path, so these cannot be recovered - the "
            "number matters for knowing how much of the history is readable, "
            "not as something to fix."
            if bad else
            "Every encrypted value read so far opened with the current key."
        ),
    }

async def stream_from_db_dec(table: str, agent: str,
                             connect_db_for_agent,
                             encrypted_fields: list = None,
                             limit: int = 1000,
                             rename_map: dict = None):
    """Stream a table out of an agent database, decrypting as it goes.

    `encrypted_fields` names that table's encrypted columns, e.g.
    `["message", "path"]`.
    """
    try:
        key_b64 = getattr(app.ctx, "fernet_key", None)
        if not key_b64:
            key_b64 = bootstrap_client.get_fernet_key()
        fernet_obj = Fernet(key_b64.encode("utf-8") if isinstance(key_b64, str) else key_b64)

        cnx = await connect_db_for_agent(agent)
        cursor = await cnx.cursor()

        query = f"SELECT * FROM {table} ORDER BY id DESC"
        if limit is not None:
            query += f" LIMIT {limit}"
        try:
            await cursor.execute(query)
        except Exception as qerr:
            err_str = str(qerr)
            if "1146" in err_str or "doesn't exist" in err_str.lower():
                await cursor.close(); await cnx.close()
                return HTTPResponse(body="[]", content_type="application/json; charset=utf-8")
            raise
        columns = [col[0] for col in cursor.description]
        rows = await cursor.fetchall()
        await cursor.close()
        await cnx.close()

        dict_rows = [dict(zip(columns, row)) for row in rows]

        if encrypted_fields is None:
            default_map = {
                "siem_events": ["message"],
                "critical_files": ["path","owner","grp","permissions","last_opened"],
                "portscan_result": ["service","product","version"],
                "packages": ["package","version"],
                "vulnerabilities_report": ["package_name","package_version","vulnerability_id","summary","details_url"],
                "events_alert": ["message","source"]
            }
            encrypted_fields = default_map.get(table, [])

        dec_rows = [decrypt_row_fields(r, encrypted_fields, fernet_obj) for r in dict_rows]

        if rename_map:
            for r in dec_rows:
                for old_k, new_k in rename_map.items():
                    if old_k in r:
                        r[new_k] = r.pop(old_k)

        body = '[' + ','.join([pyjson.dumps(r, cls=CustomEncoder, ensure_ascii=False) for r in dec_rows]) + ']'
        return HTTPResponse(body=body, content_type="application/json; charset=utf-8")

    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)

async def stream_from_db(table: str, agent: str, limit: int = 1000):
    try:
        cnx = await connect_db_for_agent(agent)
        cursor = await cnx.cursor()
        query = f"SELECT * FROM {table} ORDER BY id DESC"
        if limit is not None:
            query += f" LIMIT {limit}"
        try:
            await cursor.execute(query)
        except Exception as qerr:
            err_str = str(qerr)
            if "1146" in err_str or "doesn't exist" in err_str.lower():
                await cursor.close(); await cnx.close()
                return HTTPResponse(body="[]", content_type="application/json; charset=utf-8")
            raise
        columns = [col[0] for col in cursor.description]
        rows = await cursor.fetchall()
        await cursor.close(); await cnx.close()

        json_lines = [pyjson.dumps(dict(zip(columns, row)), cls=CustomEncoder, ensure_ascii=False) for row in rows]
        return HTTPResponse(
            body='[' + ','.join(json_lines) + ']',
            content_type="application/json; charset=utf-8"
        )
    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)


import psutil

def _server_disk_percent() -> float:
    """Return the largest disk usage percent across reachable mountpoints.

    We look at every candidate mountpoint and pick the highest non-zero
    value with non-zero total bytes. This survives the Docker Desktop /
    WSL2 case where /host_disk maps to an almost-empty WSL VM rootfs and
    reports 0% even though the real host disk is full.
    """
    candidates: list[str] = []
    seen: set[str] = set()

    def _add(p: str):
        if p and p not in seen:
            seen.add(p)
            candidates.append(p)

    if os.path.exists('/host_disk'):
        _add('/host_disk')
    _add('/')
    _add('C:\\')

    try:
        for part in psutil.disk_partitions(all=True):
            opts = (part.opts or '').lower()
            if any(tag in opts for tag in ('cdrom', 'removable')):
                continue
            mp = part.mountpoint or ''
            if not mp:
                continue
            if mp.startswith(('/proc', '/sys', '/dev', '/run')):
                continue
            _add(mp)
    except Exception:
        pass

    best = 0.0
    for c in candidates:
        try:
            usage = psutil.disk_usage(c)
            if usage.total <= 0:
                continue
            pct = float(usage.percent)
            if pct > best:
                best = pct
        except Exception:
            continue
    return best


@app.route("/server/resources", methods=["GET"])
@require_permission("read_telemetry")
async def get_server_resources(request):
    try:
        cpu = psutil.cpu_percent(interval=None)
        ram = psutil.virtual_memory().percent
        disk = await asyncio.to_thread(_server_disk_percent)
        return sanic_json({
            "status": "success",
            "cpu_usage": cpu,
            "ram_usage": ram,
            "disk_usage": disk,
            "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        })
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

@require_permission("read_telemetry")
@app.route("/<agent>/siem-events")
async def get_siem_events(request, agent):
    return await stream_from_db_dec(
        "siem_events", agent, connect_db_for_agent,
                encrypted_fields=ENCRYPTED_FIELDS_MAP["siem_events"]
    )



@require_permission("read_telemetry")
@app.route("/<agent>/events_alert")
async def get_events_alert(request, agent):
    return await stream_from_db_dec(
        "events_alert", agent, connect_db_for_agent,
                encrypted_fields=ENCRYPTED_FIELDS_MAP["events_alert"]
    )

@require_permission("read_telemetry")
@app.route("/api/agent/<agent>/inventory/hardware")
async def get_hardware_inventory(request, agent):
    fields = ENCRYPTED_FIELDS_MAP.get("hardware_inventory")
    return await stream_from_db_dec("hardware_inventory", agent, connect_db_for_agent, encrypted_fields=fields)

@require_permission("read_telemetry")
@app.route("/api/agent/<agent>/inventory/software")
async def get_software_inventory(request, agent):
    # `software_inventory`, not `packages`. The tab renders Package Name,
    # Version, Vendor and Installed Date; `packages` holds a name and a
    # version and nothing else, so two of the four columns were blank on
    # every row and had been since the tab was written. A column that is
    # always empty reads as "this host has no vendor information", which is
    # not what was happening - the agent collects vendor and install date,
    # ships them, and they land in a table nothing read.
    #
    # It is also the better inventory on Windows: `packages` comes from the
    # MSI product list, `software_inventory` from the Uninstall registry -
    # 313 programs against 167 on the host this was found on.
    #
    # `packages` stays exactly where it is. It is the vulnerability scanner's
    # input, encrypted for that reason, and its shape belongs to OSV rather
    # than to this page.
    return await stream_from_db_dec("software_inventory", agent,
                                    connect_db_for_agent)

@require_permission("read_telemetry")
@app.route("/api/agent/<agent>/inventory/network")
async def get_network_inventory(request, agent):
    # The fields were missing here and nowhere else, so the Network tab
    # rendered `enc::gAAAAAB...` where the process name should be. Ciphertext
    # in a column reads as data, not as a fault - the page looked populated
    # and was unusable. `test_encrypted_reads_decrypt` now requires every
    # route reading an encrypted table to pass its fields.
    fields = ENCRYPTED_FIELDS_MAP.get("network_connections")
    return await stream_from_db_dec("network_connections", agent,
                                    connect_db_for_agent, encrypted_fields=fields)

@require_permission("read_telemetry")
@app.route("/api/agent/<agent>/inventory/ports")
async def get_listening_ports(request, agent):
    # `network_inventory` was collected every ten minutes, shipped,
    # deduplicated, stored - and read by nothing at all. It was the last table
    # in `ALLOWED_TABLES` in that state, after `security_audit` (which had no
    # collector) and `software_inventory` (which had no reader).
    #
    # It is not the same data as the Network tab above. `network_connections`
    # is conversations already underway; this is what the host is listening
    # on, which is the question "what can be reached from outside" - and it is
    # a better answer than `portscan_result` gives, because a scan infers the
    # surface from outside while this is the kernel's own list, with the owning
    # process named.
    #
    # Plaintext, so no fields: nothing in it is sensitive on its own and the
    # port has to be indexable.
    return await stream_from_db_dec("network_inventory", agent,
                                    connect_db_for_agent)

@require_permission("read_telemetry")
@app.route("/api/agent/<agent>/fim")
async def get_fim_data(request, agent):
    fields = ENCRYPTED_FIELDS_MAP.get("fim_data")
    return await stream_from_db_dec("fim_data", agent, connect_db_for_agent, encrypted_fields=fields)

@require_permission("read_telemetry")
@app.route("/api/agent/<agent>/packages")
async def get_packages_data(request, agent):
    fields = ENCRYPTED_FIELDS_MAP.get("packages")
    return await stream_from_db_dec("packages", agent, connect_db_for_agent, encrypted_fields=fields)

@require_permission("role_create")
@require_permission("manage_users")
@app.route("/roles", methods=["POST"])
async def create_role(request):
    data = request.json or {}
    role_name = data.get("role_name")
    
    created_by = current_username(request)

    if not role_name:
        return sanic_json({
            "status": "error",
            "message": "Role name is required."
        }, status=400)

    if role_name in ["admin", "user", "guest"]:
        return sanic_json({
            "status": "error",
            "message": f"Role '{role_name}' is a system-defined role and cannot be created manually."
        }, status=403)

    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        await cursor.execute("SELECT id FROM roles WHERE role_name = %s", (role_name,))
        if await cursor.fetchone():
            await cursor.close()
            await cnx.close()
            return sanic_json({
                "status": "error",
                "message": f"Role '{role_name}' already exists."
            }, status=409)

        await cursor.execute("""
            INSERT INTO roles (role_name, created_by, created_at)
            VALUES (%s, %s, NOW())
        """, (role_name, created_by))

        role_id = cursor.lastrowid
        await cnx.commit()
        await cursor.close()
        await cnx.close()

        return sanic_json({
            "status": "success",
            "message": f"Role '{role_name}' created successfully.",
            "role_id": role_id
        }, status=201)

    except Exception as e:
        return sanic_json({
            "status": "error",
            "message": f"Database error: {str(e)}"
        }, status=500)
    

@require_permission("user_create")
@app.route("/users", methods=["POST"])
async def create_user(request):
    try:
        data_raw = request.json or {}
        user_data = CreateUserRequest(**data_raw)
        username = user_data.username
        password = user_data.password
        role = user_data.role
    except Exception as e:
        if hasattr(e, 'errors') and callable(e.errors):
            errors = e.errors()
            if len(errors) > 0 and 'msg' in errors[0]:
                clean_msg = errors[0]['msg']
                if clean_msg.startswith("Value error, "):
                    clean_msg = clean_msg[len("Value error, "):]
                return sanic_json({"status": "error", "message": clean_msg}, status=400)
                
        return sanic_json({"status": "error", "message": f"Invalid input: {str(e)}"}, status=400)
    
    created_by = current_username(request)

    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        allowed_roles = await get_allowed_roles()
        if role not in allowed_roles:
            await cursor.close(); await cnx.close()
            return sanic_json({
                "status": "error",
                "message": f"Invalid role. Allowed roles: {', '.join(allowed_roles)}"
            }, status=400)

        await cursor.execute("SELECT id FROM users WHERE username = %s", (username,))
        if await cursor.fetchone():
            await cursor.close()
            await cnx.close()
            return sanic_json({
                "status": "error",
                "message": "Username already exists."
            }, status=409)

        hashed_password = bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')

        await cursor.execute("""
            INSERT INTO users (username, password, role, created_by, created_at)
            VALUES (%s, %s, %s, %s, NOW())
        """, (username, hashed_password, role, created_by))

        user_id = cursor.lastrowid
        await cnx.commit()
        await cursor.close()
        await cnx.close()

        await audit_log(request, "CREATE_USER", username, f"Role assigned: {role}")

        return sanic_json({
            "status": "success",
            "message": f"User '{username}' created successfully with role '{role}'.",
            "user_id": user_id
        })

    except Exception as e:
        return sanic_json({
            "status": "error",
            "message": f"Database error: {str(e)}"
        }, status=500)

@require_permission("manage_users")
@app.route("/users/<user_id>/role", methods=["PUT"])
async def update_user_role(request, user_id):
    data = request.json or {}
    new_role = data.get("role")
    # Was taken from the request body, so the caller chose whose name went in
    # the audit column. Same class of bug as the old X-User-ID header.
    updated_by = current_username(request)

    allowed_roles = await get_allowed_roles()
    if new_role not in allowed_roles:
        return sanic_json({
            "status": "error",
            "message": f"Invalid role. Allowed roles: {', '.join(allowed_roles)}"
        }, status=400)

    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        await cursor.execute("SELECT username, role FROM users WHERE id = %s", (user_id,))
        user_info = await cursor.fetchone()

        if not user_info:
            await cursor.close()
            await cnx.close()
            return sanic_json({
                "status": "error",
                "message": "User not found."
            }, status=404)

        username, current_role = user_info

        if current_role == "admin" and new_role != "admin":
            await cursor.execute("SELECT COUNT(*) FROM users WHERE role = 'admin'")
            admin_count = (await cursor.fetchone())[0]

            if admin_count <= 1:
                await cursor.close()
                await cnx.close()
                return sanic_json({
                    "status": "error",
                    "message": "Cannot remove admin role from the last admin user."
                }, status=403)

        await cursor.execute("""
            UPDATE users
            SET role = %s, updated_by = %s, updated_at = NOW()
            WHERE id = %s
        """, (new_role, updated_by, user_id))

        await cnx.commit()
        await cursor.close()
        await cnx.close()

        # Their live sessions were issued against the old role, so drop them
        # rather than let a demoted user keep the privileges until timeout.
        await revoke_user_sessions(int(user_id), "role changed")

        await audit_log(request, "UPDATE_USER_ROLE", username, f"Updated role to {new_role}")

        return sanic_json({
            "status": "success",
            "message": f"User '{username}' role updated to '{new_role}'. Their active sessions were ended."
        })

    except Exception as e:
        return sanic_json({
            "status": "error",
            "message": f"Database error: {str(e)}"
        }, status=500)

@require_permission("manage_users")
@app.route("/users/<user_id>/password", methods=["PUT"])
async def admin_reset_password(request, user_id):
    data = request.json or {}
    new_password = data.get("password")
    
    if not new_password or len(new_password) < 6:
        return sanic_json({
            "status": "error",
            "message": "Password must be at least 6 characters."
        }, status=400)

    try:
        # This wrote `new_password` straight into the column. Besides storing
        # the password in the clear, it also locked the account out: `login`
        # runs bcrypt.checkpw against the stored value, which raises on
        # anything that isn't a bcrypt hash.
        hashed = bcrypt.hashpw(new_password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')

        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        # Set, not cleared. An administrator resetting someone else's password
        # has necessarily chosen it for them, so that password is known to two
        # people and must be replaced by the account holder on first use.
        await cursor.execute(
            "UPDATE users SET password = %s, must_change_password = 1 "
            "WHERE id = %s",
            (hashed, user_id))
        await cnx.commit()
        await cursor.close(); await cnx.close()

        await revoke_user_sessions(int(user_id), "password reset by admin")

        await audit_log(request, "RESET_PASSWORD", user_id, f"Password reset for user ID {user_id}")

        return sanic_json({"status": "success", "message": "User password reset. Their active sessions were ended."})
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

@require_permission("manage_users")
@app.route("/users/<user_id>", methods=["DELETE"])
async def delete_user(request, user_id):
    try:
        target_uid = int(user_id)
    except (ValueError, TypeError):
        return sanic_json({"status": "error", "message": "Invalid user ID format"}, status=400)

    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        await cursor.execute("SELECT username, role FROM users WHERE id = %s", (target_uid,))
        user = await cursor.fetchone()

        if not user:
            await cursor.close(); await cnx.close()
            return sanic_json({"status": "error", "message": "User not found"}, status=404)

        username, role = user
        if role == "admin":
            await cursor.execute("SELECT COUNT(*) FROM users WHERE role = 'admin'")
            admin_count = (await cursor.fetchone())[0]
            if admin_count <= 1:
                await cursor.close(); await cnx.close()
                return sanic_json({"status": "error", "message": "Cannot delete the last admin user."}, status=403)

        await cursor.execute("DELETE FROM users WHERE id = %s", (user_id,))
        await cnx.commit()
        await cursor.close(); await cnx.close()

        await revoke_user_sessions(target_uid, "account deleted")

        await audit_log(request, "DELETE_USER", username, f"Deleted user ID {user_id}")

        return sanic_json({
            "status": "success",
            "message": f"User '{username}' deleted successfully."
        })
    except Exception as e:
        return sanic_json({
            "status": "error",
            "message": f"Database error: {e}"
        }, status=500)

async def fetch_ldap_users():
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        await cursor.execute("SELECT ldap_host, ldap_port, bind_dn, bind_password, users_base FROM ldap_conf")
        row = await cursor.fetchone()
        await cursor.close(); await cnx.close()

        ldap_users = []

        ldap_host, ldap_port, bind_dn, encrypted_password, search_base = row
        bind_password = decrypt_password(encrypted_password)
        try:
            server = Server(ldap_host, port=ldap_port, get_info=ALL)
            conn = Connection(server, user=bind_dn, password=bind_password, auto_bind=True)

            conn.search(search_base, '(objectClass=inetOrgPerson)', attributes=['uid', 'cn'])

            for entry in conn.entries:
                ldap_users.append({
                    "username": f"{entry.uid}" if 'uid' in entry else f"(no uid)",
                    "role": "ldap",
                    "created_by": "LDAP",
                    "created_at": None
                })

        except Exception as ldap_err:
            print(f"[!] LDAP error: {ldap_err}")

        return ldap_users

    except Exception as e:
        print(f"[!] Error fetching LDAP users: {e}")
        return []


@require_permission("manage_users")
@app.route("/users", methods=["GET"])
async def list_users(request):
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        await cursor.execute("""
            SELECT id, username, role, created_by, created_at
            FROM users
            ORDER BY created_at DESC
        """)

        users = []
        for row in await cursor.fetchall():
            users.append({
                "id": row[0],
                "username": row[1],
                "role": row[2],
                "created_by": row[3],
                "created_at": row[4].strftime("%Y-%m-%d %H:%M:%S") if row[4] else None
            })

        await cursor.close(); await cnx.close()

        ldap_users = await fetch_ldap_users()
        all_users = users + ldap_users

        return sanic_json({
            "status": "success",
            "users": all_users,
            "total": len(all_users)
        })

    except Exception as e:
        return sanic_json({
            "status": "error",
            "message": f"Database error: {str(e)}"
        }, status=500)

async def get_allowed_roles():
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        await cursor.execute("SELECT role_name FROM roles")
        roles = [row[0] for row in await cursor.fetchall()]
        await cursor.close()
        await cnx.close()
        return roles
    except Exception as e:
        raise SanicException(f"Failed to fetch roles: {str(e)}", status_code=500)

@require_permission("read_telemetry")
@app.route("/<agent>/resources")
async def get_resources(request, agent):
    return await stream_from_db("resource_usage", agent)


@require_permission("read_telemetry")
@app.route("/<agent>/critical_files")
async def get_critical_files(request, agent):
    return await stream_from_db_dec(
        "critical_files", agent, connect_db_for_agent,
                encrypted_fields=ENCRYPTED_FIELDS_MAP["critical_files"]
    )


@require_permission("read_telemetry")
@app.route("/<agent>/disks")
async def get_disk_usage(request, agent):
    return await stream_from_db("disk_usage", agent)

@require_permission("read_telemetry")
@app.route("/<agent>/vulnerabilities_report")
async def get_vulnerabilities_report(request, agent):
    return await stream_from_db_dec(
        "vulnerabilities_report", agent, connect_db_for_agent,
                encrypted_fields=ENCRYPTED_FIELDS_MAP["vulnerabilities_report"]
    )

@require_permission("read_telemetry")
@app.route("/<agent>/portscan_result")
async def get_portscan_result(request, agent):
    return await stream_from_db("portscan_result", agent)

@require_permission("read_telemetry")
@app.route("/<agent>/agent_info")
async def get_agent_info(request, agent):
    return await stream_from_db("agent_info", agent)

@require_permission("read_telemetry")
@app.route("/<agent>/packages")
async def get_packages(request, agent):
    return await stream_from_db_dec(
        "packages", agent, connect_db_for_agent,
                encrypted_fields=ENCRYPTED_FIELDS_MAP["packages"]
    )

@require_permission("read_telemetry")
@app.route("/<agent>/security_audit")
async def get_security_audit(request, agent):
    """Posture findings: how the host is configured, not what happened on it.

    The last link in a chain that was broken at every point. The table existed
    in both schemas and in the ingest lists with nothing writing to it; the
    collector was added and the rows arrived; and there was still no route, so
    the data reached the server, stored correctly, decryptable, and could not
    be looked at. Each of those is the same failure one step further along.

    `finding` and `details` are encrypted at rest - they name the
    misconfiguration and where it is, which is a map for anyone who reads the
    database - so this goes through the decrypting reader. `category` and
    `severity` are plaintext on purpose: the console filters and sorts on them,
    and neither says anything a reader did not already know from the table's
    name.
    """
    return await stream_from_db_dec(
        "security_audit", agent, connect_db_for_agent,
        encrypted_fields=ENCRYPTED_FIELDS_MAP["security_audit"],
    )


@require_permission("read_telemetry")
@app.route("/<agent>/ai_logs")
async def get_ai_logs(request, agent):
    return await stream_from_db("ai_log_checker_results", agent)

@require_permission("read_telemetry")
@app.route("/<agent>/ai_insights")
async def get_agent_ai_insights(request, agent):
    """Fetch latest 20 AI insights for a specific agent"""
    return await stream_from_db("ai_analysis_results", agent, limit=20)



@require_permission("read_telemetry")
@app.route("/<agent>/docker_containers")
async def get_docker_containers(request, agent):
    return await stream_from_db("docker_containers", agent)

@require_permission("analyze_logs")
@app.route("/analyze-logs", methods=["POST"])
async def analyze_logs_all_agents(request):
    try:
        result = await analyze_all_agents_logs()
        return sanic_json(result)
    except Exception as e:
        return sanic_json({
            "success": False,
            "error": f"Error analyzing logs: {str(e)}",
            "total_logs_analyzed": 0
        }, status=500)


async def _push_recent_alerts_to_defensive(agent: str, limit: int = 25,
                                           force: bool = False) -> dict:
    """Queue the most-recent events_alert rows for defensive analysis, once each.

    Backs both the manual `/analyze-defensive/<agent>` endpoint and the
    periodic auto-sweep, so the two behave identically.

    **Why the dedup check is not optional.** This used to publish every row it
    fetched, every time it ran, with no record of what it had already sent.
    The sweep re-reads the same "most recent" rows every 300s, so each one was
    re-queued indefinitely: ai_soar_queue reached 4130 messages, all repeats
    of a handful of alerts, and ai_analysis_results grew by ~1000 rows an hour
    describing the same few events over and over. The response cache made this
    worse rather than better - each repeat was answered in 0.1s from cache, so
    the loop ran at full speed and cost nothing that would have shown up as
    slowness.

    `record_occurrence` is the same counter ingest uses, keyed by the same
    fingerprint, so an alert already analysed on the way in is not analysed
    again here.

    `force=True` is for the manual endpoint: when a human explicitly asks for
    a re-analysis they should get one, and that is a bounded, deliberate
    action rather than a loop.
    """
    rows = await asyncio.to_thread(fetch_events_alert_data, agent, limit)
    if not rows:
        return {"queued": 0, "skipped": 0, "reason": "no events_alert rows"}

    try:
        key_b64 = getattr(app.ctx, "fernet_key", None)
        if not key_b64:
            key_b64 = await asyncio.to_thread(bootstrap_client.get_fernet_key)
        fernet_obj = Fernet(key_b64.encode("utf-8") if isinstance(key_b64, str) else key_b64)
    except Exception:
        fernet_obj = None

    payloads = []
    for row in rows:
        try:
            payload = decrypt_row_fields(row, ENCRYPTED_FIELDS_MAP.get("events_alert", []), fernet_obj) if fernet_obj else dict(row)
        except Exception:
            payload = dict(row)
        payloads.append(payload)

    if force:
        to_send = [(p, None) for p in payloads]
        skipped = 0
    else:
        to_send, skipped = await asyncio.to_thread(_filter_unseen_alerts, agent, payloads)

    pushed = 0
    for payload, fingerprint in to_send:
        if fingerprint:
            payload = dict(payload, _ai_fingerprint=fingerprint)
        try:
            ok = await mq_utils.publish_to_queue(mq_utils.AI_SOAR, agent, "events_alert", payload)
            if ok:
                pushed += 1
        except Exception as e:
            print(f"[!] defensive push failed agent={agent}: {e}", flush=True)
    return {"queued": pushed, "skipped": skipped, "total_rows": len(rows)}


def _filter_unseen_alerts(agent: str, payloads: list) -> tuple[list, int]:
    """Return (unseen payloads with their fingerprints, count already seen).

    Fails open: if the dedup table cannot be reached the alerts are queued
    rather than dropped. Losing an analysis is worse than repeating one - but
    note that "fails open" is what the old behaviour did unconditionally, and
    it is why the queue ran away. Errors here are printed, not swallowed.
    """
    from core.triage import DEDUP_DDL, claim_for_analysis, compute_ai_fingerprint

    try:
        with sync_mysql_conn(f"{agent}_db") as conn:
            cursor = conn.cursor()
            try:
                cursor.execute(DEDUP_DDL)
                out, seen = [], 0
                for payload in payloads:
                    fp = compute_ai_fingerprint("events_alert", payload)
                    # claim_, not record_: the sweep re-reads the same rows on
                    # every pass and must not count each pass as a sighting.
                    if claim_for_analysis(cursor, "events_alert", fp):
                        out.append((payload, fp))
                    else:
                        seen += 1
                conn.commit()
                return out, seen
            finally:
                cursor.close()
    except Exception as e:
        print(f"[!] defensive dedup check failed agent={agent}, "
              f"queueing all {len(payloads)} alert(s): {e}", flush=True)
        return [(p, None) for p in payloads], 0


@require_permission("analyze_logs")
@app.route("/analyze-defensive/<agent>", methods=["POST"])
async def trigger_defensive_scan(request, agent):
    """Manually push the latest events_alert rows for an agent through the
    defensive AI worker. Returns immediately; results land in
    ai_analysis_results once the worker finishes (visible in AI Analysis UI).

    `force` defaults true here and false for the sweep: a person pressing the
    button has asked for a re-analysis and should get one. It is bounded by
    `limit` and by the person, which is what makes it safe - unlike the timer,
    which has neither.
    """
    data = request.json or {}
    try:
        limit = max(1, min(int(data.get("limit", 25)), 200))
    except Exception:
        limit = 25
    force = data.get("force", True) is not False
    try:
        result = await _push_recent_alerts_to_defensive(agent, limit, force=force)
        return sanic_json({"success": True, "agent": agent, **result})
    except Exception as e:
        return sanic_json({"success": False, "error": str(e)}, status=500)


# ─────────────────────── Shadow Mode (defensive AI) ───────────────────────
# When AI_SHADOW_MODE is set on the defensive worker, auto-dispatchable
# verdicts are saved as `AI_DEFENSIVE_SHADOW` insights with shadow_status
# 'pending' instead of being executed. The operator reviews them in the
# SOAR Hub and either approves (→ real SOAR action is queued) or rejects
# (→ recorded and hidden from pending list). No retention timer — proposals
# live until decided.
#
# Every approve/reject decision is written to shadow_audit_chain, a
# hash-chained append-only table that lets you prove a decision was not
# tampered with after the fact (useful for compliance audits).

_CHAIN_DDL = """
CREATE TABLE IF NOT EXISTS shadow_audit_chain (
    id            INT AUTO_INCREMENT PRIMARY KEY,
    shadow_id     INT NOT NULL,
    action        VARCHAR(64)  NOT NULL,
    target        VARCHAR(512) NOT NULL,
    decision      VARCHAR(16)  NOT NULL,
    decided_by    VARCHAR(128) NOT NULL,
    decided_at    VARCHAR(32)  NOT NULL,
    note          TEXT,
    prev_hash     CHAR(64)     NOT NULL,
    row_hash      CHAR(64)     NOT NULL,
    INDEX idx_shadow_id (shadow_id)
)
"""

async def _chain_init(cnx):
    cur = await cnx.cursor()
    try:
        await cur.execute(_CHAIN_DDL)
        await cnx.commit()
    finally:
        await cur.close()


async def _chain_append(cnx, shadow_id: int, action: str, target: str,
                         decision: str, decided_by: str, note: str = '') -> str:
    await _chain_init(cnx)
    cur = await cnx.cursor(dictionary=True)
    try:
        await cur.execute(
            "SELECT row_hash FROM shadow_audit_chain ORDER BY id DESC LIMIT 1"
        )
        last = await cur.fetchone()
        prev_hash = last['row_hash'] if last else ('0' * 64)

        # datetime.utcnow() is deprecated in 3.12 and returns a naive datetime
        # that merely *claims* to be UTC — a footgun anywhere it meets an
        # aware one. This value is hashed into the audit chain, so it should
        # be unambiguous. Same output string, explicit timezone.
        decided_at_str = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
        payload = f"{shadow_id}|{action}|{target}|{decision}|{decided_by}|{decided_at_str}|{prev_hash}"
        row_hash = hashlib.sha256(payload.encode()).hexdigest()

        await cur.execute(
            """
            INSERT INTO shadow_audit_chain
                (shadow_id, action, target, decision, decided_by, decided_at, note, prev_hash, row_hash)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
            """,
            (shadow_id, action, target, decision, decided_by, decided_at_str, note or '', prev_hash, row_hash),
        )
        await cnx.commit()
    finally:
        await cur.close()
    return row_hash


async def _chain_verify(cnx) -> dict:
    cur = await cnx.cursor(dictionary=True)
    try:
        await cur.execute("SHOW TABLES LIKE 'shadow_audit_chain'")
        if not await cur.fetchone():
            return {"valid": True, "entries": 0, "note": "chain not yet initialized"}
        await cur.execute("SELECT * FROM shadow_audit_chain ORDER BY id ASC")
        rows = await cur.fetchall()
    finally:
        await cur.close()

    if not rows:
        return {"valid": True, "entries": 0}

    prev_hash = '0' * 64
    for i, row in enumerate(rows):
        da = row['decided_at']
        da_str = da.strftime('%Y-%m-%dT%H:%M:%SZ') if hasattr(da, 'strftime') else str(da)
        payload = (
            f"{row['shadow_id']}|{row['action']}|{row['target']}|"
            f"{row['decision']}|{row['decided_by']}|{da_str}|{row['prev_hash']}"
        )
        expected = hashlib.sha256(payload.encode()).hexdigest()
        if row['row_hash'] != expected or row['prev_hash'] != prev_hash:
            return {
                "valid": False,
                "entries": len(rows),
                "tampered_at_position": i + 1,
                "chain_id": row['id'],
                "shadow_id": row['shadow_id'],
            }
        prev_hash = row['row_hash']

    return {"valid": True, "entries": len(rows), "last_hash": prev_hash}

@require_permission("read_telemetry")
@app.route("/<agent>/shadow/pending", methods=["GET"])
async def list_shadow_pending(request, agent):
    """Return pending shadow proposals for one agent, enriched with prior-execution context."""
    try:
        async with agent_conn(agent) as cnx:
            cur = await cnx.cursor(dictionary=True)
            try:
                await cur.execute(
                    """
                    SELECT a.id, a.timestamp, a.source_file, a.critical_summary, a.source_data,
                           a.proposed_action, a.proposed_target, a.shadow_status,
                           a.shadow_decided_at, a.shadow_decided_by, a.created_at,
                           (SELECT COUNT(*) FROM automations au
                            WHERE LOWER(au.action) = LOWER(a.proposed_action)
                              AND au.target = a.proposed_target) AS ctx_prev_count,
                           (SELECT MAX(au.updated_at) FROM automations au
                            WHERE LOWER(au.action) = LOWER(a.proposed_action)
                              AND au.target = a.proposed_target) AS ctx_last_seen,
                           (SELECT au.status FROM automations au
                            WHERE LOWER(au.action) = LOWER(a.proposed_action)
                              AND au.target = a.proposed_target
                            ORDER BY au.updated_at DESC LIMIT 1) AS ctx_last_status
                    FROM ai_analysis_results a
                    WHERE a.source_file = 'AI_DEFENSIVE_SHADOW'
                      AND a.shadow_status = 'pending'
                    ORDER BY a.id DESC
                    LIMIT 200
                    """
                )
                rows = list(await cur.fetchall())
            finally:
                await cur.close()
        for r in rows:
            r['agent'] = agent
            for k in ('created_at', 'timestamp', 'shadow_decided_at', 'ctx_last_seen'):
                if r.get(k) and hasattr(r[k], 'isoformat'):
                    r[k] = r[k].isoformat()
            r['ctx_prev_count'] = int(r.get('ctx_prev_count') or 0)
        return sanic_json({"success": True, "results": rows})
    except Exception as e:
        if "doesn't exist" in str(e).lower() or "1146" in str(e):
            return sanic_json({"success": True, "results": []})
        return sanic_json({"success": False, "error": str(e)}, status=500)


@require_permission("read_telemetry")
@app.route("/shadow/pending", methods=["GET"])
async def list_shadow_pending_all(request):
    """Return pending shadow proposals across every agent."""
    try:
        def get_agent_names():
            with sync_mysql_conn() as conn:
                cursor = conn.cursor()
                try:
                    cursor.execute("SHOW DATABASES")
                    return [d[0].replace('_db', '') for d in cursor.fetchall() if d[0].endswith('_db')]
                finally:
                    cursor.close()

        agents = await asyncio.to_thread(get_agent_names)
        out = []
        for ag in agents:
            try:
                async with agent_conn(ag) as cnx:
                    cur = await cnx.cursor(dictionary=True)
                    try:
                        await cur.execute(
                            """
                            SELECT a.id, a.timestamp, a.source_file, a.critical_summary, a.source_data,
                                   a.proposed_action, a.proposed_target, a.shadow_status,
                                   a.shadow_decided_at, a.shadow_decided_by, a.created_at,
                                   (SELECT COUNT(*) FROM automations au
                                    WHERE LOWER(au.action) = LOWER(a.proposed_action)
                                      AND au.target = a.proposed_target) AS ctx_prev_count,
                                   (SELECT MAX(au.updated_at) FROM automations au
                                    WHERE LOWER(au.action) = LOWER(a.proposed_action)
                                      AND au.target = a.proposed_target) AS ctx_last_seen,
                                   (SELECT au.status FROM automations au
                                    WHERE LOWER(au.action) = LOWER(a.proposed_action)
                                      AND au.target = a.proposed_target
                                    ORDER BY au.updated_at DESC LIMIT 1) AS ctx_last_status
                            FROM ai_analysis_results a
                            WHERE a.source_file = 'AI_DEFENSIVE_SHADOW'
                              AND a.shadow_status = 'pending'
                            ORDER BY a.id DESC LIMIT 100
                            """
                        )
                        for r in await cur.fetchall():
                            r['agent'] = ag
                            for k in ('created_at', 'timestamp', 'shadow_decided_at', 'ctx_last_seen'):
                                if r.get(k) and hasattr(r[k], 'isoformat'):
                                    r[k] = r[k].isoformat()
                            r['ctx_prev_count'] = int(r.get('ctx_prev_count') or 0)
                            out.append(r)
                    finally:
                        await cur.close()
            except Exception as e:
                if "doesn't exist" in str(e).lower() or "1146" in str(e):
                    continue
                print(f"[!] shadow/pending skip agent={ag}: {e}", flush=True)
                continue
        out.sort(key=lambda x: x.get('created_at') or '', reverse=True)
        return sanic_json({"success": True, "results": out})
    except Exception as e:
        return sanic_json({"success": False, "error": str(e)}, status=500)


@require_permission("read_telemetry")
@app.route("/<agent>/shadow/chain/verify", methods=["GET"])
async def verify_shadow_chain(request, agent):
    """Verify the cryptographic hash chain for one agent's shadow audit log."""
    try:
        async with agent_conn(agent) as cnx:
            result = await _chain_verify(cnx)
        result['agent'] = agent
        return sanic_json({"success": True, **result})
    except Exception as e:
        return sanic_json({"success": False, "error": str(e)}, status=500)


@require_permission("read_telemetry")
@app.route("/shadow/chain/verify", methods=["GET"])
async def verify_shadow_chain_global(request):
    """Verify shadow audit chain integrity across all agents."""
    try:
        def get_agent_names():
            with sync_mysql_conn() as conn:
                cursor = conn.cursor()
                try:
                    cursor.execute("SHOW DATABASES")
                    return [d[0].replace('_db', '') for d in cursor.fetchall() if d[0].endswith('_db')]
                finally:
                    cursor.close()

        agents = await asyncio.to_thread(get_agent_names)
        results = []
        all_valid = True
        for ag in agents:
            try:
                async with agent_conn(ag) as cnx:
                    r = await _chain_verify(cnx)
                r['agent'] = ag
                results.append(r)
                if not r.get('valid', True):
                    all_valid = False
            except Exception:
                continue
        return sanic_json({"success": True, "all_valid": all_valid, "agents": results})
    except Exception as e:
        return sanic_json({"success": False, "error": str(e)}, status=500)


@require_permission("manage_soar")
@app.route("/<agent>/shadow/<insight_id:int>/approve", methods=["POST"])
async def approve_shadow(request, agent, insight_id):
    """Approve a pending shadow proposal → dispatch the proposed SOAR action."""
    try:
        async with agent_conn(agent) as cnx:
            cur = await cnx.cursor(dictionary=True)
            try:
                await cur.execute(
                    """
                    SELECT id, proposed_action, proposed_target, shadow_status, critical_summary
                    FROM ai_analysis_results WHERE id = %s
                    """,
                    (insight_id,),
                )
                row = await cur.fetchone()
            finally:
                await cur.close()

        if not row:
            return sanic_json({"success": False, "error": "shadow proposal not found"}, status=404)
        if row.get('shadow_status') != 'pending':
            return sanic_json(
                {"success": False, "error": f"proposal already {row.get('shadow_status') or 'finalized'}"},
                status=409,
            )

        action = row.get('proposed_action') or ''
        target = row.get('proposed_target') or ''
        if not action or not target:
            return sanic_json({"success": False, "error": "proposal has no action/target"}, status=400)

        actor = ''
        try:
            actor = (request.ctx.session.get('username') if getattr(request.ctx, 'session', None) else '') or ''
        except Exception:
            actor = ''
        if not actor:
            actor = request.headers.get('X-Forwarded-User') or 'operator'

        soar_result = await call_agent_soar(
            agent,
            action=action,
            target=target,
            comment=f"Shadow approved by {actor}: insight#{insight_id}",
        )

        chain_hash = ''
        async with agent_conn(agent) as cnx:
            cur = await cnx.cursor()
            try:
                await cur.execute(
                    """
                    UPDATE ai_analysis_results
                    SET shadow_status = 'approved',
                        shadow_decided_at = NOW(),
                        shadow_decided_by = %s,
                        critical_summary = CONCAT(critical_summary, '\n[APPROVED] ', %s, ' by ', %s)
                    WHERE id = %s
                    """,
                    (actor, action.upper(), actor, insight_id),
                )
                await cnx.commit()
            finally:
                await cur.close()
            chain_hash = await _chain_append(cnx, insight_id, action, target, 'approved', actor)

        return sanic_json({
            "success": True,
            "insight_id": insight_id,
            "action": action,
            "target": target,
            "dispatched": bool(soar_result.get('ok')),
            "soar_result": soar_result,
            "chain_hash": chain_hash,
        })
    except Exception as e:
        return sanic_json({"success": False, "error": str(e)}, status=500)


@require_permission("manage_soar")
@app.route("/<agent>/shadow/<insight_id:int>/reject", methods=["POST"])
async def reject_shadow(request, agent, insight_id):
    """Reject a pending shadow proposal → record decision, no action taken."""
    data = request.json or {}
    note = (data.get('note') or '').strip()[:240]
    try:
        actor = ''
        try:
            actor = (request.ctx.session.get('username') if getattr(request.ctx, 'session', None) else '') or ''
        except Exception:
            actor = ''
        if not actor:
            actor = request.headers.get('X-Forwarded-User') or 'operator'

        action_for_chain = ''
        target_for_chain = ''
        async with agent_conn(agent) as cnx:
            cur = await cnx.cursor(dictionary=True)
            try:
                await cur.execute(
                    "SELECT proposed_action, proposed_target FROM ai_analysis_results WHERE id = %s",
                    (insight_id,),
                )
                meta = await cur.fetchone()
                action_for_chain = (meta or {}).get('proposed_action') or ''
                target_for_chain = (meta or {}).get('proposed_target') or ''
            finally:
                await cur.close()

            cur2 = await cnx.cursor()
            try:
                await cur2.execute(
                    """
                    UPDATE ai_analysis_results
                    SET shadow_status = 'rejected',
                        shadow_decided_at = NOW(),
                        shadow_decided_by = %s,
                        critical_summary = CONCAT(critical_summary, '\n[REJECTED] by ', %s, %s)
                    WHERE id = %s AND shadow_status = 'pending'
                    """,
                    (actor, actor, (f' ({note})' if note else ''), insight_id),
                )
                affected = cur2.rowcount
                await cnx.commit()
            finally:
                await cur2.close()

            if affected and action_for_chain:
                await _chain_append(cnx, insight_id, action_for_chain, target_for_chain, 'rejected', actor, note)

        if not affected:
            return sanic_json({"success": False, "error": "not found or already finalized"}, status=404)
        return sanic_json({"success": True, "insight_id": insight_id, "actor": actor})
    except Exception as e:
        return sanic_json({"success": False, "error": str(e)}, status=500)


DEFENSIVE_SWEEP_INTERVAL = int(os.getenv("AI_DEFENSIVE_SWEEP_SEC", "300"))
DEFENSIVE_SWEEP_BATCH = int(os.getenv("AI_DEFENSIVE_SWEEP_BATCH", "10"))


async def _defensive_auto_sweep():
    """Background loop: pick up alerts that reached the database without going
    through the ingest AI path, and queue each one for defensive analysis once.

    This is a backstop, not a generator of activity. Its previous docstring
    said it existed "so the LLM keeps producing visible commentary even when
    nothing fresh is streaming through ingest", and it delivered exactly that
    - the same handful of alerts re-analysed every five minutes so the AI
    page would not look idle. An idle AI page when nothing is happening is the
    correct display.

    With the dedup check in `_push_recent_alerts_to_defensive`, a sweep over
    alerts that have all been seen queues nothing and prints nothing.
    """
    await asyncio.sleep(30)
    while True:
        try:
            def _list_agents():
                with sync_mysql_conn() as conn:
                    cursor = conn.cursor()
                    try:
                        cursor.execute("SHOW DATABASES")
                        return [r[0].replace('_db', '') for r in cursor.fetchall() if r[0].endswith('_db')]
                    finally:
                        cursor.close()

            agents = await asyncio.to_thread(_list_agents)
            for ag in agents:
                try:
                    res = await _push_recent_alerts_to_defensive(ag, DEFENSIVE_SWEEP_BATCH)
                    if res.get("queued"):
                        print(f"[AI-Sweep] defensive pushed {res['queued']} new alert(s) "
                              f"for {ag} ({res.get('skipped', 0)} already analysed)", flush=True)
                except Exception as e:
                    print(f"[!] defensive sweep agent={ag}: {e}", flush=True)
        except Exception as e:
            print(f"[!] defensive sweep loop error: {e}", flush=True)
        await asyncio.sleep(DEFENSIVE_SWEEP_INTERVAL)


@app.before_server_start
async def start_defensive_sweep(app):
    # Only in worker 0-0, like every other background loop here. Without this
    # the sweep runs once per Sanic worker, and each copy queues the same
    # alerts - multiplying the publish rate by the worker count for work that
    # is meant to happen once.
    worker_name = os.environ.get("SANIC_WORKER_NAME", "")
    if not ("0-0" in worker_name or not worker_name):
        return
    if os.getenv("AI_DEFENSIVE_SWEEP_ENABLED", "1") == "1":
        app.add_task(_defensive_auto_sweep())
        print(f"[*] Defensive AI auto-sweep enabled (every {DEFENSIVE_SWEEP_INTERVAL}s, batch={DEFENSIVE_SWEEP_BATCH})", flush=True)

@require_permission("analyze_logs")
@app.route("/analyze-logs/<agent>", methods=["POST"])
async def analyze_logs_single_agent(request, agent):
    """Analyze SIEM logs for a specific agent using AI"""
    data = request.json or {}
    limit = int(data.get('limit', 100))
    
    try:
        result = await analyze_agent_logs(agent, limit)
        return sanic_json(result)
    except Exception as e:
        return sanic_json({
            "success": False,
            "error": f"Error analyzing logs for agent {agent}: {str(e)}",
            "logs_analyzed": 0
        }, status=500)

@app.route("/api/analyze-selected/<agent>", methods=["POST"])
@require_permission("analyze_logs")
async def analyze_selected_logs_route(request, agent):
    """Analyze a specific set of logs selected from the UI"""
    data = request.json or {}
    logs = data.get('logs', [])
    
    if not logs:
        return sanic_json({"success": False, "error": "No logs provided"}, status=400)
        
    try:
        result = await analyze_selected_logs(agent, logs)
        return sanic_json(result)
    except Exception as e:
        return sanic_json({"success": False, "error": str(e)}, status=500)

@app.route("/api/ai-insights/all", methods=["GET"])
@require_permission("read_telemetry")
async def get_all_ai_insights(request):
    """Fetch latest AI insights from all agent databases"""
    try:
        def get_agent_names():
            with sync_mysql_conn() as conn:
                cursor = conn.cursor()
                try:
                    cursor.execute("SHOW DATABASES")
                    return [db[0].replace('_db', '') for db in cursor.fetchall() if db[0].endswith('_db')]
                finally:
                    cursor.close()

        agents = await asyncio.to_thread(get_agent_names)
        per_agent = max(1, min(int(request.args.get("per_agent", 100)), 500))
        total_limit = max(1, min(int(request.args.get("limit", 500)), 2000))

        async def _for_agent(agent):
            async with agent_conn(agent) as cnx:
                acur = await cnx.cursor(dictionary=True)
                try:
                    # This used to run an information_schema lookup and an
                    # ALTER TABLE on every request, for every agent, just to
                    # find out whether the table existed. Both are expensive,
                    # the DDL is redundant — save_ai_results owns that
                    # migration — and a missing table is cheaper to detect by
                    # letting the SELECT fail.
                    #
                    # `source_data` is excluded: it holds the entire raw log
                    # per insight, and the feed does not render it. It is
                    # fetched per insight by the View Source modal instead.
                    # LEFT JOIN, not INNER: an insight whose event was never
                    # deduplicated (or predates the dedup table) still has to
                    # appear. `occurrences` is how many times this exact event
                    # recurred — the repeats attach here instead of each
                    # costing another inference.
                    await acur.execute(
                        """SELECT r.id, r.timestamp, r.source_file, r.critical_summary,
                                  r.verdict, r.severity, r.confidence, r.model,
                                  r.proposed_action, r.proposed_target, r.shadow_status,
                                  r.created_at,
                                  COALESCE(d.occurrences, 1) AS occurrences,
                                  d.last_seen AS last_occurrence
                             FROM ai_analysis_results r
                             LEFT JOIN ai_dedup d ON d.fingerprint = r.fingerprint
                            ORDER BY r.created_at DESC, r.id DESC
                            LIMIT %s""",
                        (per_agent,),
                    )
                    rows = await acur.fetchall()
                except Exception as e:
                    if "1146" in str(e) or "doesn't exist" in str(e).lower():
                        return []
                    raise
                finally:
                    await acur.close()

            for r in rows:
                r['agent'] = agent
                for field in ('created_at', 'timestamp', 'last_occurrence'):
                    if r.get(field) and hasattr(r[field], 'isoformat'):
                        r[field] = r[field].isoformat()
                if r.get('confidence') is not None:
                    r['confidence'] = float(r['confidence'])
                if r.get('occurrences') is not None:
                    r['occurrences'] = int(r['occurrences'])
            return rows

        buckets = await gather_over_agents(_for_agent, agents)
        all_results = [row for bucket in buckets for row in bucket]

        sorted_results = sorted(
            all_results,
            key=lambda x: x.get('created_at') or '',
            reverse=True,
        )[:total_limit]

        return sanic_json({
            "success": True,
            "results": sorted_results,
            "agents_scanned": agents,
            "agents_with_insights": sorted({r['agent'] for r in sorted_results}),
        })
        
    except Exception as e:
        return sanic_json({"success": False, "error": str(e)}, status=500)

@app.get("/<agent>/ai_insights/<insight_id:int>/source")
@require_permission("read_telemetry")
async def get_insight_source(request, agent, insight_id):
    """The raw log one insight was produced from.

    Split out of the list response because `source_data` is a LONGTEXT holding
    the full event, and shipping it for every row made the AI feed several
    megabytes when only the rows an operator actually opens need it.
    """
    try:
        async with agent_conn(agent) as cnx:
            cur = await cnx.cursor(dictionary=True)
            try:
                await cur.execute(
                    "SELECT id, source_file, source_data, payload, timestamp "
                    "FROM ai_analysis_results WHERE id = %s",
                    (insight_id,),
                )
                row = await cur.fetchone()
            finally:
                await cur.close()

        if not row:
            return sanic_json({"status": "error", "message": "Insight not found"}, status=404)

        return sanic_json(
            {"status": "success", "insight": row},
            dumps=lambda o: pyjson.dumps(o, ensure_ascii=False, cls=CustomEncoder),
        )
    except Exception as e:
        return _db_object_error(e)


@require_permission("analyze_logs")
@app.route("/ai-config/<agent>", methods=["GET"])
async def get_ai_config(request, agent):
    """Get AI configuration for a specific agent (Defaults to Ollama if not set)"""
    try:
        config = await load_ai_config(agent)
        if not config:
            config = {
                'model_name': OLLAMA_MODEL,
                'endpoint': OLLAMA_BASE_URL,
                'api_key': 'ollama',
                'updated_at': None
            }
        
        is_ollama_key = config.get('api_key') == 'ollama'
        safe_config = {
            'model_name': config.get('model_name') or OLLAMA_MODEL,
            'endpoint': config.get('endpoint') or OLLAMA_BASE_URL,
            'has_api_key': bool(config.get('api_key')) and not is_ollama_key,
            'updated_at': config.get('updated_at')
        }
        return sanic_json({"success": True, "config": safe_config})
    except Exception as e:
        return sanic_json({"success": False, "error": str(e)}, status=500)

class ActionType(Enum):
    BLOCK_IP = "block_ip"
    UNBLOCK_IP = "unblock_ip"
    DISABLE_USER = "disable_user"
    ENABLE_USER = "enable_user"
    KILL_PROCESS = "kill_process"
    ISOLATE_HOST = "isolate_host"
    QUARANTINE_FILE = "quarantine_file"
    DELETE_FILE = "delete_file"
    RUN_COMMAND = "run_cmd"
    RESTART_SERVICE = "restart_service"
    LOCK_MACHINE = "lock_machine"
    TAIL_LOG = "tail_log"

AUTOMATION_ALLOWED_ACTIONS = [
    "block_ip", "unblock_ip", "disable_user", "enable_user", "kill_process", 
    "isolate_host", "quarantine_file", "delete_file", "run_cmd",
    "restart_service", "lock_machine", "tail_log"
]

@require_permission("manage_agent")
@app.route("/<agent>/config/<cfg_type>", methods=["GET"])
async def get_agent_config_proxy(request, agent, cfg_type):
    """Proxy request to agent to get a YAML config file"""
    if cfg_type not in ["rules", "log_paths", "file_scan"]:
        return sanic_json({"status": "error", "message": "Invalid config type"}, status=400)

    body, status = await _agent_proxy(agent, "GET", f"/config/{cfg_type}")
    return sanic_json(body, status=status)

@require_permission("manage_agent")
@app.post("/<agent>/config/<cfg_type>/validate")
async def validate_agent_config(request, agent, cfg_type):
    """Check a config without pushing it, so the editor can lint as you type."""
    data = request.json or {}
    issues = config_validation.validate(cfg_type, data.get("content", ""))
    return sanic_json({
        "status": "success",
        "valid": not config_validation.blocking(issues),
        "issues": [i.to_dict() for i in issues],
    })


@require_permission("manage_agent")
@app.route("/<agent>/config/<cfg_type>", methods=["POST"])
async def set_agent_config_proxy(request, agent, cfg_type):
    """Validate a YAML config, then push it to the agent.

    This forwarded the request body unread. A typo therefore reached the
    sensor, where it surfaced as the agent quietly not detecting things any
    more — nothing in the UI looked wrong, which is the worst way for a
    security tool to fail. Nothing reaches an endpoint now unless it parses,
    has the right shape, and has regexes that actually compile.
    """
    if cfg_type not in config_validation.CONFIG_TYPES:
        return sanic_json({"status": "error", "message": "Invalid config type"}, status=400)

    data = request.json or {}
    content = data.get("content", "")

    issues = config_validation.validate(cfg_type, content)
    errors = config_validation.blocking(issues)
    if errors:
        await audit_log(request, "CONFIG_PUSH_REJECTED", f"{agent}/{cfg_type}",
                        f"{len(errors)} blocking issue(s)")
        return sanic_json({
            "status": "error",
            "message": f"Config rejected: {len(errors)} problem(s) found. Nothing was sent to the agent.",
            "issues": [i.to_dict() for i in issues],
        }, status=400)

    body, status = await _agent_proxy(agent, "POST", f"/config/{cfg_type}", data)

    # Audit what happened, not what was attempted: a push that never reached
    # the agent is not a config change, and recording it as one leaves an
    # audit trail claiming a sensor was reconfigured when it was not.
    await audit_log(
        request,
        "CONFIG_PUSH" if status < 400 else "CONFIG_PUSH_FAILED",
        f"{agent}/{cfg_type}",
        f"{len(content)} bytes, {len(issues)} warning(s)"
        + ("" if status < 400 else f" - {body.get('message', status)}"))

    # Warnings survive a successful push so the operator still sees them.
    if isinstance(body, dict) and issues:
        body["issues"] = [i.to_dict() for i in issues]
    return sanic_json(body, status=status)

@require_permission("manage_system")
@app.route("/ai-config/<agent>", methods=["POST"])
async def set_ai_config(request, agent):
    """Set AI configuration (agent param currently unused but kept for routing)"""
    data = request.json or {}
    
    required_fields = ['model_name', 'endpoint']
    for field in required_fields:
        if not data.get(field):
            return sanic_json({"success": False, "error": f"Missing required field: {field}"}, status=400)
    
    api_key = data.get('api_key')
    
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        
        if not api_key:
            if "ollama" in data.get('endpoint', '').lower() or "11434" in data.get('endpoint', ''):
                api_key = "ollama"
            else:
                await cursor.execute("SELECT api_key FROM ai_config ORDER BY updated_at DESC LIMIT 1")
                row = await cursor.fetchone()
                if row:
                    api_key = row[0]

        await cursor.execute("""
            INSERT INTO ai_config (model_name, api_key, endpoint, updated_at)
            VALUES (%s, %s, %s, NOW())
            ON DUPLICATE KEY UPDATE
                model_name = VALUES(model_name),
                api_key = VALUES(api_key),
                endpoint = VALUES(endpoint),
                updated_at = NOW()
        """, (data['model_name'], api_key, data['endpoint']))
        
        await cnx.commit()
        await cursor.close()
        await cnx.close()
        
        return sanic_json({"success": True, "message": "AI configuration updated successfully"})
        
    except Exception as e:
        return sanic_json({"success": False, "error": f"Database error: {str(e)}"}, status=500)


@require_permission("role_create")
@app.route("/roles/assign-permissions", methods=["POST"])
async def assign_permissions_to_role(request):
    data = request.json or {}
    role_name = data.get("role_name")
    permissions = data.get("permissions", []) 
    all_permissions = data.get("all_permissions", False)

    if not role_name:
        return sanic_json({"status": "error", "message": "role_name is required"}, status=400)

    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        await cursor.execute("SELECT id FROM roles WHERE role_name = %s", (role_name,))
        role_row = await cursor.fetchone()
        if not role_row:
            await cursor.close(); await cnx.close()
            return sanic_json({"status": "error", "message": f"Role '{role_name}' not found."}, status=404)

        role_id = role_row[0]

        if all_permissions:
            await cursor.execute("""
                INSERT IGNORE INTO role_permissions (role_id, permission_id)
                SELECT %s, id FROM permissions
            """, (role_id,))
        else:
            for perm_name in permissions:
                await cursor.execute("""
                    INSERT IGNORE INTO role_permissions (role_id, permission_id)
                    SELECT %s, id FROM permissions WHERE name = %s
                """, (role_id, perm_name))

        await cnx.commit()
        await cursor.close(); await cnx.close()

        return sanic_json({
            "status": "success",
            "message": f"Permissions assigned to role '{role_name}' successfully."
        })

    except Exception as e:
        return sanic_json({
            "status": "error",
            "message": f"Failed to assign permissions: {str(e)}"
        }, status=500)
    
@app.route("/ai-analysis-status/<agent>")
@require_permission("read_telemetry")
async def get_ai_analysis_status(request, agent):
    try:
        db_name = _agent_db_name(agent)
        def _query():
            with sync_mysql_conn(db_name) as conn:
                cursor = conn.cursor(dictionary=True)
                try:
                    cursor.execute("SELECT COUNT(*) as total FROM siem_events")
                    total_logs = cursor.fetchone()['total']
                    cursor.execute("SELECT COUNT(*) as analyzed FROM siem_events WHERE ai_analyzed = TRUE")
                    analyzed_logs = cursor.fetchone()['analyzed']
                    cursor.execute("SELECT COUNT(*) as pending FROM siem_events WHERE (ai_analyzed IS NULL OR ai_analyzed = FALSE)")
                    pending_logs = cursor.fetchone()['pending']
                    cursor.execute("SELECT * FROM ai_log_checker_results ORDER BY created_at DESC LIMIT 10")
                    recent_results = cursor.fetchall()
                    return total_logs, analyzed_logs, pending_logs, recent_results
                finally:
                    cursor.close()
        total_logs, analyzed_logs, pending_logs, recent_results = await asyncio.to_thread(_query)
        
        return sanic_json({
            "success": True,
            "agent": agent,
            "total_logs": total_logs,
            "analyzed_logs": analyzed_logs,
            "pending_logs": pending_logs,
            "analysis_percentage": round((analyzed_logs / total_logs * 100) if total_logs > 0 else 0, 2),
            "recent_results": recent_results
        })
        
    except Exception as e:
        return sanic_json({"success": False, "error": str(e)}, status=500)
    

@require_permission("set_email_config")
@app.route("/send-critical-alerts", methods=["POST"])
async def send_critical_alerts_endpoint(request):
    """Manually trigger sending critical alerts for all agents"""
    try:
        total_alerts = dispatch_all_critical_alerts()
        return sanic_json({
            "status": "success",
            "message": f"Processed {total_alerts} critical alerts",
            "alerts_sent": total_alerts
        })
    except Exception as e:
        return sanic_json({
            "status": "error",
            "message": f"Error sending critical alerts: {str(e)}"
        }, status=500)

@require_permission("set_email_config")
@app.route("/send-critical-alerts/<agent>", methods=["POST"])
async def send_critical_alerts_agent(request, agent):
    """Manually trigger sending critical alerts for a specific agent"""
    try:
        alerts_count = dispatch_critical_alerts(agent)
        return sanic_json({
            "status": "success",
            "message": f"Processed {alerts_count} critical alerts for agent {agent}",
            "alerts_sent": alerts_count
        })
    except Exception as e:
        return sanic_json({
            "status": "error",
            "message": f"Error sending critical alerts for agent {agent}: {str(e)}"
        }, status=500)

@app.get("/email-templates")
@require_permission("set_email_config")
async def list_email_templates(request):
    """Templates with their bodies, for the editor.

    `/<agent>/notifications/templates` returns names only and exists for the
    playbook node picker. Editing content needs the whole row, and until now
    there was no way to see or change one outside the database.
    """
    try:
        async with userdb_conn() as cnx:
            cur = await cnx.cursor(dictionary=True)
            try:
                await cur.execute(
                    "SELECT id, template_name, subject_template, body_template, updated_at "
                    "FROM email_templates ORDER BY template_name"
                )
                rows = await cur.fetchall()
            finally:
                await cur.close()
        return sanic_json(
            {"status": "success", "templates": rows, "default_name": DEFAULT_ALERT_TEMPLATE},
            dumps=lambda o: pyjson.dumps(o, ensure_ascii=False, cls=CustomEncoder),
        )
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)


@app.post("/email-templates")
@require_permission("set_email_config")
async def upsert_email_template(request):
    """Create or replace a template, keyed on its name."""
    data = request.json or {}
    name = (data.get("template_name") or "").strip()
    subject = (data.get("subject_template") or "").strip()
    body = (data.get("body_template") or "").strip()

    missing = [f for f, v in (("template_name", name), ("subject_template", subject),
                              ("body_template", body)) if not v]
    if missing:
        return sanic_json(
            {"status": "error", "message": f"Missing required field(s): {', '.join(missing)}"},
            status=400,
        )

    try:
        async with userdb_conn() as cnx:
            cur = await cnx.cursor()
            try:
                await cur.execute(
                    """INSERT INTO email_templates (template_name, subject_template, body_template)
                       VALUES (%s, %s, %s)
                       ON DUPLICATE KEY UPDATE
                           subject_template = VALUES(subject_template),
                           body_template = VALUES(body_template)""",
                    (name, subject, body),
                )
                await cnx.commit()
            finally:
                await cur.close()
        await audit_log(request, "EMAIL_TEMPLATE_SAVE", name, "")
        return sanic_json({"status": "success", "message": f"Template '{name}' saved."})
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)


@app.delete("/email-templates/<template_id:int>")
@require_permission("set_email_config")
async def delete_email_template(request, template_id):
    try:
        async with userdb_conn() as cnx:
            cur = await cnx.cursor()
            try:
                await cur.execute("SELECT template_name FROM email_templates WHERE id = %s", (template_id,))
                row = await cur.fetchone()
                if not row:
                    return sanic_json({"status": "error", "message": "Template not found"}, status=404)
                name = row[0]
                # Deleting the fallback would silently stop alert mail for every
                # agent that has no template of its own.
                if name == DEFAULT_ALERT_TEMPLATE:
                    return sanic_json({
                        "status": "error",
                        "message": f"'{name}' is the fallback used when an agent has no template of "
                                   f"its own. Edit it instead of deleting it.",
                    }, status=400)
                await cur.execute("DELETE FROM email_templates WHERE id = %s", (template_id,))
                await cnx.commit()
            finally:
                await cur.close()
        await audit_log(request, "EMAIL_TEMPLATE_DELETE", name, "")
        return sanic_json({"status": "success", "message": f"Template '{name}' deleted."})
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)


@require_permission("set_email_config")
@require_permission("manage_system")
@app.route("/email-config", methods=["GET"])
async def get_email_config(request):
    """Get current email configuration status"""
    config = get_mail_config()
    return sanic_json({
        "enabled": config['enabled'],
        "smtp_server": config.get('smtp_server', 'Not configured') if config['enabled'] else 'Not configured',
        "from_addr": config.get('from_addr', 'Not configured') if config['enabled'] else 'Not configured',
        "to_addr": config.get('to_addr', 'Not configured') if config['enabled'] else 'Not configured'
    })

@require_permission("set_email_config")
@app.route("/email-config", methods=["POST"])
async def set_email_config(request):
    """Set email configuration"""
    data = request.json or {}
    
    required_fields = ['smtp_server', 'smtp_port', 'email_from', 'email_to']
    for field in required_fields:
        if not data.get(field):
            return sanic_json({"status": "error", "message": f"Missing required field: {field}"}, status=400)
    
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        
        query = """
            INSERT INTO email_config (smtp_server, smtp_port, smtp_user, smtp_password, 
                                    smtp_use_tls, email_from, email_to, updated_at)
            VALUES (%s, %s, %s, %s, %s, %s, %s, NOW())
            ON DUPLICATE KEY UPDATE
                smtp_server = VALUES(smtp_server),
                smtp_port = VALUES(smtp_port),
                smtp_user = VALUES(smtp_user),
                smtp_password = VALUES(smtp_password),
                smtp_use_tls = VALUES(smtp_use_tls),
                email_from = VALUES(email_from),
                email_to = VALUES(email_to),
                updated_at = NOW()
        """
        
        await cursor.execute(query, (
            data['smtp_server'],
            data['smtp_port'],
            data.get('smtp_user'),
            data.get('smtp_password'),
            data.get('smtp_use_tls', False),
            data['email_from'],
            data['email_to']
        ))
        
        await cnx.commit()
        await cursor.close()
        await cnx.close()
        
        return sanic_json({"status": "success", "message": "Email configuration updated successfully"})
        
    except Exception as e:
        return sanic_json({"status": "error", "message": f"Database error: {str(e)}"}, status=500)

def encrypt_password(password: str) -> str:
    return fernet.encrypt(password.encode()).decode()

def decrypt_password(encrypted: str) -> str:
    return fernet.decrypt(encrypted.encode()).decode()

async def log_login_attempt(username, auth_type, status, reason, ip_address):
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        await cursor.execute("""
            INSERT INTO login_logs (username, auth_type, status, reason, ip_address)
            VALUES (%s, %s, %s, %s, %s)
        """, (username, auth_type, status, reason, ip_address))
        await cnx.commit()
        await cursor.close()
        await cnx.close()
    except Exception as e:
        print(f"[!] Error: {e}")


@app.route("/<agent>/soar_actions/<action_id>/resolve", methods=["PATCH"])
@require_permission("manage_soar")
async def resolve_soar_action(request, agent, action_id):
    if not is_soar_enabled():
        return _soar_block_response()
    try:
        data = request.json or {}
        raw_ts = (data.get("resolved_at") or "").strip()
        if raw_ts:
            ts = raw_ts.replace("T", " ").rstrip("Z")
        else:
            ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

        new_status = (data.get("status") or "RESOLVED").upper()

        cnx = await connect_db_for_agent(agent)
        cursor = await cnx.cursor()

        await cursor.execute(
            "UPDATE soar_actions SET resolved_at = %s, status = %s WHERE id = %s",
            (ts, new_status, action_id)
        )
        await cnx.commit()
        updated = cursor.rowcount
        await cursor.close()

        if not updated:
            await cnx.close()
            return sanic_json({"status": "error", "message": "SOAR action not found"}, status=404)

        cur2 = await cnx.cursor()
        await cur2.execute("SELECT * FROM soar_actions WHERE id = %s", (action_id,))
        columns = [c[0] for c in cur2.description]
        row = await cur2.fetchone()
        await cur2.close(); await cnx.close()

        body = pyjson.dumps(dict(zip(columns, row)), cls=CustomEncoder, ensure_ascii=False)
        return HTTPResponse(body=body, content_type="application/json; charset=utf-8")

    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)


@app.route("/<agent>/soar_actions/<action_id>/send", methods=["PATCH"])
@require_permission("manage_soar")
async def mark_soar_action_sent(request, agent, action_id):
    if not is_soar_enabled():
        return _soar_block_response()
    try:
        data = request.json or {}
        sent_bool = _as_bool(data.get("sent", True))
        sent_val = 1 if sent_bool else 0

        cnx = await connect_db_for_agent(agent)
        cursor = await cnx.cursor()
        await cursor.execute(
            "UPDATE soar_actions SET sent = %s WHERE id = %s",
            (sent_val, action_id)
        )
        await cnx.commit()
        updated = cursor.rowcount
        await cursor.close()

        if not updated:
            await cnx.close()
            return sanic_json({"status": "error", "message": "SOAR action not found"}, status=404)

        cur2 = await cnx.cursor()
        await cur2.execute("SELECT * FROM soar_actions WHERE id = %s", (action_id,))
        columns = [c[0] for c in cur2.description]
        row = await cur2.fetchone()
        await cur2.close(); await cnx.close()

        body = pyjson.dumps(dict(zip(columns, row)), cls=CustomEncoder, ensure_ascii=False)
        return HTTPResponse(body=body, content_type="application/json; charset=utf-8")

    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)


@app.route("/<agent>/soar_actions/<action_id>", methods=["DELETE"])
@require_permission("manage_soar")
async def delete_soar_action(request, agent, action_id):
    if not is_soar_enabled():
        return _soar_block_response()
    try:
        cnx = await connect_db_for_agent(agent)
        cursor = await cnx.cursor()
        await cursor.execute("DELETE FROM soar_actions WHERE id = %s", (action_id,))
        await cnx.commit()
        deleted = cursor.rowcount
        await cursor.close(); await cnx.close()

        if not deleted:
            return sanic_json({"status": "error", "message": "SOAR action not found"}, status=404)

        return sanic_json({"status": "success", "message": f"SOAR action {action_id} deleted"})
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

@require_permission("manage_soar")
@app.post("/<agent>/automations/<auto_id:int>/execute")
async def execute_automation(request, agent, auto_id):
    if not is_soar_enabled():
        return _soar_block_response()

    try:
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor(dictionary=True)
        await cur.execute("SELECT * FROM automations WHERE id=%s AND device=%s", (auto_id, agent))
        rec = await cur.fetchone()
        await cur.close(); await cnx.close()
        if not rec:
            return sanic_json({"error": "automation not found"}, status=404)

        if rec["action"] not in AUTOMATION_ALLOWED_ACTIONS:
            return sanic_json({"error": f"action not allowed: {rec['action']}"}, status=400)

        if rec["status"] not in ("pending", "paused", "failed", "active"):
            return sanic_json({"error": f"cannot execute in status={rec['status']}"}, status=409)

        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute("UPDATE automations SET status='active', updated_at=NOW() WHERE id=%s", (auto_id,))
        await cnx.commit(); await cur.close(); await cnx.close()

        prefix = f"automation#{rec['id']}"
        result = await call_agent_soar(
            agent,
            action=rec["action"],
            target=rec["target"],
            comment=f"{prefix} | {rec.get('comment') or ''}".strip(),
            event_id=rec.get("event_id"),
            # Already queued - this row is what is being executed. Leaving the
            # default on made every dispatch enqueue its own successor.
            background_queue=False,
        )

        auto_status = "completed" if result.get("ok") else "failed"
        final_comment = (rec.get("comment") or "").strip()
        if result.get("message"):
            add = f"{prefix} | {result['message']}"
            final_comment = (f"{final_comment} | {add}" if final_comment else add)[:500]

        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute(
            "UPDATE automations SET status=%s, comment=%s, updated_at=NOW() WHERE id=%s",
            (auto_status, final_comment, rec["id"])
        )
        await cnx.commit(); await cur.close(); await cnx.close()

        return sanic_json({
            "status": "ok",
            "automation": {"id": rec["id"], "status": auto_status, "comment": final_comment},
            "agent_response": result,
        })

    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)


@require_permission("manage_soar")
@app.post("/<agent>/automations/run-due")
async def run_due_automations(request, agent):

    if not is_soar_enabled():
        return _soar_block_response()
    try:
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor(dictionary=True)
        await cur.execute(
            """
            SELECT * FROM automations
            WHERE device=%s AND status='pending' AND `timestamp` <= NOW()
            ORDER BY `timestamp` ASC, id ASC
            """,
            (agent,)
        )
        rows = await cur.fetchall()
        await cur.close(); await cnx.close()

        results = []
        for rec in rows or []:
            cnx = await connect_db_for_agent(agent)
            cur = await cnx.cursor()
            await cur.execute("UPDATE automations SET status='active', updated_at=NOW() WHERE id=%s", (rec["id"],))
            await cnx.commit(); await cur.close(); await cnx.close()

            prefix = f"automation#{rec['id']}"
            try:
                r = await call_agent_soar(
                    agent,
                    action=rec["action"],
                    target=rec["target"],
                    comment=f"{prefix} | {rec.get('comment') or ''}".strip(),
                    event_id=rec.get("event_id"),
                    # Already queued - see the note in run_due_automations.
                    background_queue=False,
                )
                auto_status = "completed" if r.get("ok") else "failed"
                final_comment = (rec.get("comment") or "").strip()
                if r.get("message"):
                    add = f"{prefix} | {r['message']}"
                    final_comment = (f"{final_comment} | {add}" if final_comment else add)[:500]
            except Exception as ex:
                r = {"ok": False, "error": str(ex)}
                auto_status = "failed"
                final_comment = (rec.get("comment") or "")[:500]

            cnx = await connect_db_for_agent(agent)
            cur = await cnx.cursor()
            await cur.execute(
                "UPDATE automations SET status=%s, comment=%s, updated_at=NOW() WHERE id=%s",
                (auto_status, final_comment, rec["id"])
            )
            await cnx.commit(); await cur.close(); await cnx.close()

            results.append({"id": rec["id"], "status": auto_status, "agent_response": r})

        ok = sum(1 for r in results if r["status"] == "completed")
        fail = sum(1 for r in results if r["status"] == "failed")
        return sanic_json({"status": "ok", "count": len(results), "completed": ok, "failed": fail, "results": results})

    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)


@app.get("/<agent>/soar_actions")
@require_permission("manage_soar")
async def get_soar_actions_api(request, agent):
    """
    Alias for /automations to support frontend SoarHub.
    """
    try:
        return await fixed_list_automations(request, agent)
    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)


@app.post("/automations/report", name="report_automation_gen")
@app.post("/<agent>/automations/report", name="report_automation_agent")
@app.post("/api/agents/<agent>/automations/report", name="report_automation_api")
async def report_automation_result(request, agent=None):
    """Where an agent reports the result of an action it was told to take.

    Same treatment as report_automation_result_by_id: the identity comes from
    the key rather than from `metadata.agent` in the body, and a task can only
    move from outstanding to finished.

    The two `except Exception: pass` blocks are also gone. A row lives in
    `soar_actions` or in `automations`, not both, so one of the two statements
    was always expected to fail - and swallowing both meant a report that
    updated nothing was indistinguishable from one that worked. The rowcounts
    are counted instead, and a report that matched nothing is a 404.
    """
    try:
        data = request.json or {}
        task_id = data.get("task_id")
        status = str(data.get("status", "success")).strip().lower()
        output = data.get("output", "")

        identity = await _validate_agent_auth(request)
        if identity is None or identity == "*":
            return sanic_json({"error": "unauthorized"}, status=401)
        if agent and _agent_name_forms(identity)[1] != _agent_name_forms(agent)[1]:
            print(f"[!] agent key for {identity!r} reported for {agent!r}", flush=True)
            return sanic_json({"error": "unauthorized"}, status=401)
        agent = identity

        if not task_id:
            return sanic_json({"error": "Missing task_id"}, status=400)
        if status not in _TERMINAL_AUTOMATION_STATES:
            return sanic_json(
                {"error": f"status must be one of {sorted(_TERMINAL_AUTOMATION_STATES)}"},
                status=400)

        async with agent_conn(agent) as cnx:
            cur = await cnx.cursor()
            try:
                touched = 0
                for table, value in (("soar_actions", status.upper()),
                                     ("automations", status)):
                    try:
                        await cur.execute(
                            f"UPDATE {table} SET status = %s, comment = %s, "
                            f"updated_at = NOW() WHERE id = %s "
                            f"AND LOWER(status) IN ('pending','active')",
                            (value, output[:500], task_id))
                        touched += cur.rowcount
                    except Exception as e:
                        # A missing table is normal on an agent that has never
                        # run that kind of action. Anything else is reported.
                        if "1146" not in str(e) and "doesn't exist" not in str(e).lower():
                            print(f"[!] report_automation_result {table}: {e}", flush=True)
                await cnx.commit()
            finally:
                await _close_async_quiet(cur)

        if not touched:
            return sanic_json(
                {"error": "no outstanding task with that id for this agent"},
                status=404)
        return sanic_json({"status": "ok"})
    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)





def _to_mysql_ts(raw: str | None) -> str | None:
    if not raw:
        return None
    s = str(raw).strip()
    s = s.replace("T", " ")
    if s.endswith("Z"):
        s = s[:-1]
    try:
        dt = datetime.fromisoformat(s)
        return dt.strftime("%Y-%m-%d %H:%M:%S")
    except Exception:
        return s[:19]

@require_permission("manage_soar")
@app.get("/<agent>/automations")
async def fixed_list_automations(request, agent):
    """
    List both automations (playbooks/queued) and soar_actions (manual/pushed) for an agent.
    Merged into a unified view for the SOAR UI.
    """
    q = request.args
    status_filter = (q.get("status") or "").strip().lower()
    search = (q.get("search") or q.get("q") or "").strip().lower()
    order = (q.get("order") or "DESC").upper()
    if order not in ("ASC", "DESC"):
        order = "DESC"

    try:
        limit = int(q.get("limit", 500))
        limit = max(1, min(limit, 5000))
    except Exception:
        limit = 500

    try:
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()

        all_data = []

        try:
            await cur.execute("SELECT id, device, event_id, action, target, comment, status, `timestamp`, payload, created_at, updated_at, 'automation' as source FROM automations WHERE device = %s ORDER BY id DESC LIMIT %s", (agent, limit))
            cols = [d[0] for d in cur.description]
            all_data.extend([dict(zip(cols, r)) for r in await cur.fetchall()])
        except Exception:
            await cur.execute("SELECT id, device, event_id, action, target, comment, status, `timestamp`, payload, created_at, created_at as updated_at, 'automation' as source FROM automations WHERE device = %s ORDER BY id DESC LIMIT %s", (agent, limit))
            cols = [d[0] for d in cur.description]
            all_data.extend([dict(zip(cols, r)) for r in await cur.fetchall()])

        rows_soar = []
        try:
            await cur.execute("SELECT id, action, target, comment, status, `timestamp`, created_at, updated_at, 'manual' as source FROM soar_actions ORDER BY id DESC LIMIT %s", (limit,))
            cols = [d[0] for d in cur.description]
            rows_soar = [dict(zip(cols, r)) for r in await cur.fetchall()]
        except Exception:
            await cur.execute("SELECT id, action, target, comment, status, `timestamp`, created_at, created_at as updated_at, 'manual' as source FROM soar_actions ORDER BY id DESC LIMIT %s", (limit,))
            cols = [d[0] for d in cur.description]
            rows_soar = [dict(zip(cols, r)) for r in await cur.fetchall()]

        await cur.close(); await cnx.close()

        for r in rows_soar:
            r["device"] = agent
            r["event_id"] = 0
            r["payload"] = None
            if "updated_at" not in r: r["updated_at"] = r.get("created_at")
            all_data.append(r)

        def _ensure_ts(val):
            if val is None: return 0.0
            if hasattr(val, "timestamp"): return float(val.timestamp())
            if isinstance(val, (int, float)): return float(val)
            if isinstance(val, str):
                try:
                    from datetime import datetime
                    dt = datetime.fromisoformat(val.replace(" ", "T"))
                    return float(dt.timestamp())
                except Exception:
                    try: return float(val)
                    except Exception: return 0.0
            return 0.0

        filtered = []
        for r in all_data:
            if status_filter and status_filter != str(r.get("status", "")).lower():
                continue
            if search:
                text = f"{r.get('action', '')} {r.get('target', '')} {r.get('comment', '')}".lower()
                if search not in text:
                    continue
            
            r["timestamp"] = _ensure_ts(r.get("timestamp"))
            r["updated_at"] = _ensure_ts(r.get("updated_at"))
            r["created_at"] = _ensure_ts(r.get("created_at"))

            if r.get("payload") and isinstance(r["payload"], str):
                try:
                    r["payload"] = pyjson.loads(r["payload"])
                except ValueError:
                    # A bare `except:` here also caught KeyboardInterrupt and
                    # SystemExit. Leave the payload as the string it is; the
                    # UI renders it either way.
                    pass

            filtered.append(r)

        try:
            filtered.sort(key=lambda x: float(x.get("timestamp") or 0.0), reverse=(order == "DESC"))
        except Exception as e:
            print(f"[!] SORT ERROR in fixed_list_automations: {e}")
            raise e
            
        return sanic_json(filtered[:limit])

    except Exception as e:
        print(f"[!] Error in fixed_list_automations: {e}")
        return sanic_json({"error": str(e)}, status=500)

@require_permission("manage_soar")
@app.post("/<agent>/automations")
async def create_automation(request, agent):
    data = request.json or {}

    try:
        event_id = int(data.get("event_id"))
    except Exception:
        return sanic_json({"error": "event_id (int) required"}, status=400)

    action  = (data.get("action") or "").strip()
    target  = (data.get("target") or "").strip()
    status  = (data.get("status") or "pending").strip()
    comment = (data.get("comment") or "").strip()
    ts_in   = _to_mysql_ts(data.get("timestamp"))

    if not action or not target:
        return sanic_json({"error": "action and target required"}, status=400)
    if action not in AUTOMATION_ALLOWED_ACTIONS:
        return sanic_json({"error": f"action not allowed: {action}"}, status=400)
    if status not in AUTOMATION_ALLOWED_STATUSES:
        return sanic_json({"error": f"status not allowed: {status}"}, status=400)

    payload = data.get("payload")
    if payload is not None and not isinstance(payload, (dict, list, str)):
        return sanic_json({"error": "payload must be object/array/string"}, status=400)
    payload_str = None
    if isinstance(payload, (dict, list)):
        payload_str = pyjson.dumps(payload, ensure_ascii=False)
    elif isinstance(payload, str):
        payload_str = payload

    try:
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()

        await cur.execute("""
            INSERT INTO automations
                (device, event_id, action, target, comment, status, `timestamp`, payload, created_at, updated_at)
            VALUES
                (%s, %s, %s, %s, %s, %s, COALESCE(%s, NOW()), %s, NOW(), NOW())
        """, (agent, event_id, action, target, comment, status, ts_in, payload_str))
        new_id = cur.lastrowid
        await cnx.commit()
        await cur.close()

        cur2 = await cnx.cursor()
        await cur2.execute("SELECT id, device, event_id, action, target, comment, status, `timestamp`, payload, created_at, updated_at FROM automations WHERE id=%s AND device=%s", (new_id, agent))
        row = await cur2.fetchone()
        cols = [c[0] for c in cur2.description]
        await cur2.close(); await cnx.close()

        d = dict(zip(cols, row))
        if d.get("payload"):
            try:
                d["payload"] = pyjson.loads(d["payload"]) if isinstance(d["payload"], str) else d["payload"]
            except Exception:
                pass

        return sanic_json(
                d,
                status=201,
                dumps=lambda obj: pyjson.dumps(obj, ensure_ascii=False, cls=CustomEncoder),
            )
    
    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)

@require_permission("manage_soar")
@app.put("/<agent>/automations/<auto_id:int>")
async def update_automation(request, agent, auto_id):
    data = request.json or {}
    fields, params = [], []

    if "event_id" in data:
        try:
            ev = int(data.get("event_id"))
            fields.append("event_id = %s"); params.append(ev)
        except Exception:
            return sanic_json({"error": "event_id must be int"}, status=400)

    if "action" in data:
        action = (data.get("action") or "").strip()
        if action not in AUTOMATION_ALLOWED_ACTIONS:
            return sanic_json({"error": f"action not allowed: {action}"}, status=400)
        fields.append("action = %s"); params.append(action)

    if "target" in data:
        fields.append("target = %s"); params.append((data.get("target") or "").strip())

    if "comment" in data:
        fields.append("comment = %s"); params.append((data.get("comment") or "").strip())

    if "status" in data:
        status = (data.get("status") or "").strip()
        if status not in AUTOMATION_ALLOWED_STATUSES:
            return sanic_json({"error": f"status not allowed: {status}"}, status=400)
        fields.append("status = %s"); params.append(status)

    if "timestamp" in data:
        fields.append("`timestamp` = %s"); params.append(_to_mysql_ts(data.get("timestamp")) or datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

    if "payload" in data:
        payload = data.get("payload")
        if payload is not None and not isinstance(payload, (dict, list, str)):
            return sanic_json({"error": "payload must be object/array/string"}, status=400)
        payload_str = pyjson.dumps(payload, ensure_ascii=False) if isinstance(payload, (dict, list)) else (payload if isinstance(payload, str) else None)
        fields.append("payload = %s"); params.append(payload_str)

    if not fields:
        return sanic_json({"error": "no updatable fields provided"}, status=400)

    fields.append("updated_at = NOW()")

    try:
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        sql = f"UPDATE automations SET {', '.join(fields)} WHERE id=%s AND device=%s"
        params.extend([auto_id, agent])
        await cur.execute(sql, tuple(params))
        await cnx.commit()
        updated = cur.rowcount
        await cur.close()

        if not updated:
            await cnx.close()
            return sanic_json({"error": "automation not found"}, status=404)

        cur2 = await cnx.cursor()
        await cur2.execute("SELECT id, device, event_id, action, target, comment, status, `timestamp`, payload, created_at, updated_at FROM automations WHERE id=%s AND device=%s", (auto_id, agent))
        row = await cur2.fetchone()
        cols = [c[0] for c in cur2.description]
        await cur2.close(); await cnx.close()

        d = dict(zip(cols, row))
        if d.get("payload"):
            try:
                d["payload"] = pyjson.loads(d["payload"]) if isinstance(d["payload"], str) else d["payload"]
            except Exception:
                pass
        return sanic_json(
                    d,
                    status=201,
                    dumps=lambda obj: pyjson.dumps(obj, ensure_ascii=False, cls=CustomEncoder),
                )
    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)

@app.patch("/<agent>/automations/<auto_id:int>/status")
@require_permission("manage_soar")
async def patch_automation_status(request, agent, auto_id):
    data = request.json or {}
    new_status = (data.get("status") or "").strip()
    if new_status not in AUTOMATION_ALLOWED_STATUSES:
        return sanic_json({"error": f"status not allowed: {new_status}"}, status=400)

    extra_comment = (data.get("comment") or "").strip()

    try:
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()

        if extra_comment:
            await cur.execute("""
                UPDATE automations
                SET status=%s, comment=CONCAT(COALESCE(comment,''), CASE WHEN LENGTH(COALESCE(comment,''))>0 THEN ' | ' ELSE '' END, %s),
                    updated_at=NOW()
                WHERE id=%s AND device=%s
            """, (new_status, extra_comment, auto_id, agent))
        else:
            await cur.execute("""
                UPDATE automations
                SET status=%s, updated_at=NOW()
                WHERE id=%s AND device=%s
            """, (new_status, auto_id, agent))

        await cnx.commit()
        updated = cur.rowcount
        await cur.close()

        if not updated:
            await cnx.close()
            return sanic_json({"error": "automation not found"}, status=404)

        cur2 = await cnx.cursor()
        await cur2.execute("SELECT id, device, event_id, action, target, comment, status, `timestamp`, payload, created_at, updated_at FROM automations WHERE id=%s AND device=%s", (auto_id, agent))
        row = await cur2.fetchone()
        cols = [c[0] for c in cur2.description]
        await cur2.close(); await cnx.close()

        d = dict(zip(cols, row))
        if d.get("payload"):
            try:
                d["payload"] = pyjson.loads(d["payload"]) if isinstance(d["payload"], str) else d["payload"]
            except Exception:
                pass
        return sanic_json(
                d,
                status=201,
                dumps=lambda obj: pyjson.dumps(obj, ensure_ascii=False, cls=CustomEncoder),
            )
    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)


@require_permission("manage_soar")
@app.delete("/<agent>/automations/<auto_id:int>")
async def delete_automation(request, agent, auto_id):
    try:
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute("DELETE FROM automations WHERE id=%s AND device=%s", (auto_id, agent))
        await cnx.commit()
        deleted = cur.rowcount
        await cur.close(); await cnx.close()

        if not deleted:
            return sanic_json({"error": "automation not found"}, status=404)

        return sanic_json({"status": "success", "message": f"automation {auto_id} deleted"})
    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)

@app.post("/<agent>/automations/<auto_id:int>/run")
@require_permission("manage_soar")
async def run_automation_alias(request, agent, auto_id):
    return await execute_automation(request, agent, auto_id)


@app.post("/<agent>/automations/validate-target")
@require_permission("manage_soar")
async def validate_automation_target(request, agent):
    if not is_soar_enabled():
        return _soar_block_response()
    data = request.json or {}
    action = (data.get("action") or "").strip()
    target = (data.get("target") or "").strip()

    if action not in AUTOMATION_ALLOWED_ACTIONS:
        return sanic_json({"ok": False, "error": f"action not allowed: {action}"}, status=400)

    try:
        if action == ActionType.BLOCK_IP.value:
            if not _is_valid_ipv4(target):
                return sanic_json({"ok": False, "error": "invalid IPv4"}, status=400)
            return sanic_json({"ok": True})

        elif action == ActionType.DISABLE_USER.value:
            if not _is_valid_username(target):
                return sanic_json({"ok": False, "error": "invalid username"}, status=400)
            return sanic_json({"ok": True})

        return sanic_json({"ok": False, "error": f"action not implemented: {action}"}, status=501)

    except Exception as e:
        return sanic_json({"ok": False, "error": str(e)}, status=500)


def _set_session_cookie(resp, raw_token: str):
    """Attach the session cookie.

    Sanic 23.3+ exposes `add_cookie()`; older releases only have the mapping
    API. `sanic` is unpinned in requirements.txt, so support both.
    """
    add = getattr(resp.cookies, "add_cookie", None)
    if add is not None:
        add(
            session_store.SESSION_COOKIE, raw_token,
            path="/", httponly=True, samesite=SESSION_COOKIE_SAMESITE,
            secure=SESSION_COOKIE_SECURE,
            max_age=session_store.cookie_max_age(),
        )
        return
    resp.cookies[session_store.SESSION_COOKIE] = raw_token
    c = resp.cookies[session_store.SESSION_COOKIE]
    c["path"] = "/"
    c["httponly"] = True
    c["samesite"] = SESSION_COOKIE_SAMESITE
    c["secure"] = SESSION_COOKIE_SECURE
    c["max-age"] = session_store.cookie_max_age()


def _clear_session_cookie(resp):
    delete = getattr(resp.cookies, "delete_cookie", None)
    if delete is not None:
        # `secure` must mirror what _set_session_cookie used. Sanic's
        # delete_cookie defaults it to True, and a Secure deletion cookie
        # delivered over plain http is dropped by the browser — so on an
        # http deployment the cookie would survive logout. The session is
        # revoked server-side either way, but leaving a live-looking cookie
        # in the browser is not a state worth shipping.
        delete(session_store.SESSION_COOKIE, path="/", secure=SESSION_COOKIE_SECURE)
        return
    resp.cookies[session_store.SESSION_COOKIE] = ""
    resp.cookies[session_store.SESSION_COOKIE]["max-age"] = 0


async def _issue_session(request, *, user_id: int, username: str,
                         role: str | None, auth_type: str) -> str:
    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            raw = await session_store.create(
                cur,
                user_id=user_id, username=username, role=role,
                auth_type=auth_type, ip=_client_ip(request),
                user_agent=request.headers.get("User-Agent"),
            )
            await cnx.commit()
        finally:
            await cur.close()
    return raw


async def _upsert_ldap_user(username: str, role: str) -> int:
    """Give an LDAP identity a local `users` row.

    Sessions, RBAC and audit attribution all key off the same integer id, so an
    LDAP login needs one too — before this, the LDAP branch returned a user
    object with no `id` at all and the frontend threw on `user.id.toString()`.
    The password column gets an unusable placeholder: `bcrypt.checkpw` raises
    on it, which the local branch already treats as "fall through to LDAP".
    """
    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            await cur.execute("SELECT id FROM users WHERE username = %s LIMIT 1", (username,))
            row = await cur.fetchone()
            if row:
                await cur.execute("UPDATE users SET role = %s WHERE id = %s", (role, row[0]))
                await cnx.commit()
                return int(row[0])
            await cur.execute(
                "INSERT INTO users (username, password, role, created_by) VALUES (%s, %s, %s, 'ldap')",
                (username, "!ldap-no-local-password", role),
            )
            await cnx.commit()
            await cur.execute("SELECT id FROM users WHERE username = %s LIMIT 1", (username,))
            row = await cur.fetchone()
            return int(row[0]) if row else 0
        finally:
            await cur.close()


@app.route("/login", methods=["GET", "POST"])
async def login(request):
    if request.method == "GET":
        return await response.file("./frontend/dist/index.html")

    try:
        data_raw = request.json or {}
        login_data = LoginRequest(**data_raw)
        username = login_data.username
        password = login_data.password
    except Exception as e:
        return response.json({"status": "error", "message": f"Invalid input: {str(e)}"}, status=400)

    ip = _client_ip(request)

    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        # Before the password check. `login_logs` already held a row for every
        # failure and nothing read it: bcrypt at cost 12 was the only brake on
        # online guessing, which is detection without response.
        locked = await login_guard.check(cursor, username, ip)
        if locked:
            await cursor.close(); await cnx.close()
            await log_login_attempt(username, "local", "failure",
                                    f"locked out: {locked}", ip)
            print(f"[auth] lockout {username!r} from {ip}: {locked}", flush=True)
            # The conclusion, not every attempt. One failed password is not an
            # incident and storing it as one is how a security view becomes
            # noise; the threshold has already been crossed by the time this
            # branch runs, and that is the thing worth putting in front of
            # somebody.
            await _platform_event("LOGIN_LOCKOUT", subject=username,
                                  source_ip=ip, detail=locked)
            return response.json({
                "status": "error",
                "message": "Too many failed attempts. Try again later.",
            }, status=429)

        await cursor.execute(
            "SELECT id, username, password, created_at, role, "
            "COALESCE(must_change_password, 0) FROM users "
            "WHERE username = %s LIMIT 1",
            (username,),
        )
        row = await cursor.fetchone()
        await cursor.close()
        await cnx.close()

        if row:
            user_id, db_username, db_password, created_at, db_role, must_change = row
            if bcrypt.checkpw(password.encode('utf-8'), db_password.encode('utf-8')):
                await log_login_attempt(username, "local", "success", "", ip)

                print(f"[+] Local login successful for user: {username}")
                # Past this point the credentials are correct, so nothing may
                # fall through to LDAP.
                #
                # It used to. The whole local branch sat inside one `try`
                # whose `except` means "not a local user, try the directory",
                # so a failure *after* the password matched - issuing the
                # session, reading permissions - was caught by it, the LDAP
                # path was tried, and the operator got 401. With a working
                # database this never happened. With a pool that failed to
                # build at startup it happened every time, and told the person
                # with the right password that it was wrong.
                # The second factor, if this account has a confirmed one.
                #
                # Before the session, not after it. Issuing one and
                # "upgrading" it once a code arrives leaves a window in which
                # a single factor is a logged-in operator, and every bug in
                # that window is a bypass.
                async with userdb_conn() as f_cnx:
                    f_cur = await f_cnx.cursor()
                    try:
                        needs_second = await _totp_required(f_cur, user_id)
                    finally:
                        await f_cur.close()
                if needs_second:
                    pending = await _begin_second_factor(
                        request, user_id=user_id, username=db_username,
                        auth_type="local")
                    await log_login_attempt(username, "local", "pending",
                                            "password accepted, second factor "
                                            "required", ip)
                    return response.json({
                        "status": "second_factor_required",
                        "token": pending,
                        "expires_in": totp_store.PENDING_TTL_SECONDS,
                        "message": "Enter the code from your authenticator.",
                    })

                try:
                    user_perms = await get_user_permissions(user_id)
                    raw_token = await _issue_session(
                        request, user_id=user_id, username=db_username,
                        role=db_role, auth_type="local",
                    )
                except Exception as e:
                    print(f"[!] {username} authenticated but the session could "
                          f"not be created: {e}", flush=True)
                    await log_login_attempt(username, "local", "failure",
                                            f"authenticated, session failed: {e}", ip)
                    return response.json({
                        "status": "error",
                        "message": "Your password was accepted, but this server "
                                   "could not start a session. This is a server "
                                   "fault, not a credential one - check that it "
                                   "can reach its database.",
                    }, status=503)
                resp = response.json({
                    "status": "success",
                    "message": "Login successful (local).",
                    "user": {
                        "id": user_id,
                        "username": db_username,
                        "created_at": created_at.strftime("%Y-%m-%d %H:%M:%S") if hasattr(created_at, 'strftime') else str(created_at),
                        "auth_type": "local",
                        # Was hardcoded to `"admin" if username == "admin"`,
                        # which silently ignored the role column.
                        "role": db_role,
                        "permissions": user_perms,
                        # So the console can say why it is empty. The
                        # middleware refuses every other route while this is
                        # set, and without the flag reaching the UI the only
                        # symptom was blank pages.
                        "must_change_password": bool(must_change),
                    }
                })
                _set_session_cookie(resp, raw_token)
                return resp
            else:
                await log_login_attempt(username, "local", "failure", "Invalid local password", ip)
                print(f"[-] Local password mismatch for user: {username}")
                return response.json({"status": "error", "message": "Invalid username or password."}, status=401)
        else:
            print(f"[-] User {username} not found in local DB. Trying LDAP...")

    except Exception as e:
        await log_login_attempt(username, "Local", "failure", f"{e} Looking for LDAP", ip)

    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        await cursor.execute("""
            SELECT ldap_host, ldap_port, bind_dn, bind_password, users_base, group_base, login_filter 
            FROM ldap_conf
            LIMIT 1
        """)
        row = await cursor.fetchone()
        await cursor.close()
        await cnx.close()

        if not row:
            await log_login_attempt(username, "ldap", "failure", "No LDAP config", ip)
            return response.json({"status": "error", "message": "Invalid username or password."}, status=401)

        ldap_host, ldap_port, bind_dn, encrypted_password, users_base, group_base, login_filter = row
        bind_password = decrypt_password(encrypted_password)
        tls_config = Tls(validate=ssl.CERT_REQUIRED)
        max_retries = 3

        server = Server(ldap_host, port=ldap_port, use_ssl=True, tls=tls_config, connect_timeout=3)

        for attempt in range(max_retries):
            try:
                admin_conn = Connection(server, user=bind_dn, password=bind_password, auto_bind=True)

                # Escaped, not interpolated raw. `login_filter % username` with
                # a username of `*)(uid=*` produces a filter that matches every
                # entry, so the directory returns the first user in the tree
                # and authentication becomes "did you type anything".
                #
                # The pattern on LoginRequest already rejects the metacharacters;
                # this is the second layer, because the filter template is
                # operator-configurable and a future one could interpolate
                # somewhere the pattern does not cover.
                search_filter = login_filter % escape_filter_chars(username)
                admin_conn.search(users_base, search_filter, attributes=["cn"])
                if not admin_conn.entries:
                    admin_conn.unbind()
                    await log_login_attempt(username, "ldap", "failure", "User not found", ip)
                    return response.json({"status": "error", "message": "Invalid username or password."}, status=401)

                user_dn = admin_conn.entries[0].entry_dn

                admin_conn.search(
                    search_base=group_base,
                    # The DN comes from the directory rather than the user,
                    # but it is still interpolated into a filter, and a DN can
                    # legitimately contain parentheses.
                    search_filter=f"(member={escape_filter_chars(user_dn)})",
                    attributes=["cn"]
                )
                group_dns = [entry.entry_dn for entry in admin_conn.entries]
                admin_conn.unbind()

                try:
                    user_conn = Connection(server, user=user_dn, password=password, auto_bind=True)
                    user_conn.unbind()
                except LDAPBindError:
                    await log_login_attempt(username, "ldap", "failure", "Wrong password", ip)
                    return response.json({"status": "error", "message": "Invalid username or password."}, status=401)

                app_role = "user"
                try:
                    cnx = await connect_userdb()
                    cursor = await cnx.cursor()
                    for group_dn in group_dns:
                        await cursor.execute(
                            "SELECT app_role FROM ldap_role_mappings WHERE ldap_group_dn = %s LIMIT 1",
                            (group_dn,)
                        )
                        r = await cursor.fetchone()
                        if r:
                            app_role = r[0]
                            print(app_role)
                            break
                    await cursor.close()
                    await cnx.close()
                except Exception as e:
                    # Swallowed silently before. When this lookup fails the
                    # user is admitted with the default role instead of the
                    # one their LDAP group maps to, which reads as a
                    # permissions bug months later rather than as an outage
                    # now.
                    print(f"[!] LDAP role lookup failed for {username!r}: {e}",
                          flush=True)

                user_perms = await get_role_permissions(app_role)
                ldap_uid = await _upsert_ldap_user(username, app_role)

                # The same gate as the local branch, and it has to be here
                # too. A second factor that protects one login path and not
                # the other protects nothing: an attacker with a password
                # picks the path without it.
                async with userdb_conn() as f_cnx:
                    f_cur = await f_cnx.cursor()
                    try:
                        needs_second = await _totp_required(f_cur, ldap_uid)
                    finally:
                        await f_cur.close()
                if needs_second:
                    pending = await _begin_second_factor(
                        request, user_id=ldap_uid, username=username,
                        auth_type="ldap")
                    await log_login_attempt(username, "ldap", "pending",
                                            "password accepted, second factor "
                                            "required", ip)
                    return response.json({
                        "status": "second_factor_required",
                        "token": pending,
                        "expires_in": totp_store.PENDING_TTL_SECONDS,
                        "message": "Enter the code from your authenticator.",
                    })

                await log_login_attempt(username, "ldap", "success", "", ip)
                raw_token = await _issue_session(
                    request, user_id=ldap_uid, username=username,
                    role=app_role, auth_type="ldap",
                )
                resp = response.json({
                    "status": "success",
                    "message": "Login successful (LDAP).",
                    "user": {
                        "id": ldap_uid,
                        "username": username,
                        "auth_type": "ldap",
                        "role": app_role,
                        "permissions": user_perms
                    }
                })
                _set_session_cookie(resp, raw_token)
                return resp

            except LDAPSocketOpenError as e:
                if attempt == max_retries - 1:
                    await log_login_attempt(username, "ldap", "failure", f"Ldap server unreachable: {e}", ip)
                    return response.json({"status": "error", "message": "Invalid username or password."}, status=504)
                await asyncio.sleep(1)

    except Exception as e:
        await log_login_attempt(username, "ldap", "failure", f"Exception: {e}", ip)
        return response.json({"status": "error", "message": "Invalid username or password."}, status=401)

    await log_login_attempt(username, "combined", "failure", "All methods failed", ip)
    return response.json({"status": "error", "message": "Invalid username or password."}, status=401)


@app.post("/logout")
async def logout(request):
    """Revoke the current session server-side and clear the cookie.

    Public so that a request carrying an already-dead session still clears the
    browser's cookie instead of bouncing off the 401 in `authenticate`.
    """
    raw_token = request.cookies.get(session_store.SESSION_COOKIE)
    if raw_token:
        try:
            async with userdb_conn() as cnx:
                cur = await cnx.cursor()
                try:
                    await session_store.revoke(cur, raw_token)
                    await cnx.commit()
                finally:
                    await cur.close()
        except Exception as e:
            print(f"[!] Logout revoke failed: {e}")

    resp = sanic_json({"status": "success", "message": "Logged out."})
    _clear_session_cookie(resp)
    return resp



def _sync_ldap_bind(ldap_host, ldap_port, bind_dn, bind_password, use_ssl: bool):
    tls_config = Tls(validate=ssl.CERT_REQUIRED)

    server = Server(
        ldap_host,
        port=int(ldap_port),
        use_ssl=use_ssl,
        tls=tls_config,
        connect_timeout=3,   
        get_info=None        
    )

    conn = Connection(
        server,
        user=bind_dn,
        password=bind_password,
        auto_bind=True,
        raise_exceptions=True,
        receive_timeout=3
    )
    try:
        pass
    finally:
        conn.unbind()

@app.route("/ldap/test-connection", methods=["POST"])
@require_permission("manage_system")
async def test_ldap_connection(request):
    data = request.json or {}
    ldap_host = data.get("ldap_host")
    ldap_port = data.get("ldap_port")
    bind_dn = data.get("bind_dn")
    bind_password = data.get("bind_password")
    use_ssl = bool(data.get("use_ssl", True))

    if not all([ldap_host, ldap_port, bind_dn, bind_password]):
        return response.json({"status": "error", "message": "Missing LDAP parameters"}, status=400)
    try:
        ldap_port = int(ldap_port)
        if not (1 <= ldap_port <= 65535):
            raise ValueError()
    except Exception:
        return response.json({"status": "error", "message": "Invalid ldap_port"}, status=400)

    overall_timeout = float(data.get("timeout", 5))

    try:
        await asyncio.wait_for(
            asyncio.to_thread(_sync_ldap_bind, ldap_host, ldap_port, bind_dn, bind_password, use_ssl),
            timeout=overall_timeout
        )
        return response.json({"status": "success", "message": "LDAP connection successful"})

    except asyncio.TimeoutError:
        return response.json(
            {"status": "error", "message": f"LDAP connection timed out (>{overall_timeout}s)"},
            status=504
        )
    except LDAPBindError as e:
        return response.json(
            {"status": "error", "message": f"LDAP bind failed: {e}"},
            status=401
        )
    except LDAPSocketOpenError as e:
        return response.json(
            {"status": "error", "message": f"LDAP socket error: {e}"},
            status=502
        )
    except Exception as e:
        return response.json(
            {"status": "error", "message": f"LDAP connection failed: {e}"},
            status=500
        )


@require_permission("role_create")
@app.route("/roles/<role_id>", methods=["PUT"])
async def update_role(request, role_id):
    data = request.json or {}
    new_role_name = data.get("role_name")
    permissions = data.get("permissions", [])
    
    updated_by = current_username(request)

    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        await cursor.execute("SELECT role_name FROM roles WHERE id = %s", (role_id,))
        result = await cursor.fetchone()
        if not result:
            await cursor.close()
            await cnx.close()
            return sanic_json({
                "status": "error",
                "message": "Role not found."
            }, status=404)

        current_role_name = result[0]

        if current_role_name in ["admin", "user", "guest"]:
            await cursor.close()
            await cnx.close()
            return sanic_json({
                "status": "error",
                "message": f"System-defined role '{current_role_name}' cannot be edited."
            }, status=403)

        if new_role_name and new_role_name != current_role_name:
            await cursor.execute("SELECT id FROM roles WHERE role_name = %s AND id != %s", (new_role_name, role_id))
            if await cursor.fetchone():
                await cursor.close()
                await cnx.close()
                return sanic_json({
                    "status": "error",
                    "message": f"Role name '{new_role_name}' already exists."
                }, status=409)

            await cursor.execute("""
                UPDATE roles 
                SET role_name = %s, updated_by = %s, updated_at = NOW()
                WHERE id = %s
            """, (new_role_name, updated_by, role_id))

            await cursor.execute("""
                UPDATE users 
                SET role = %s 
                WHERE role = %s
            """, (new_role_name, current_role_name))

        if permissions is not None:
            await cursor.execute("DELETE FROM role_permissions WHERE role_id = %s", (role_id,))

            if permissions:
                placeholders = ','.join(['%s'] * len(permissions))
                await cursor.execute(f"SELECT name FROM permissions WHERE name IN ({placeholders})", permissions)
                valid_permissions = [row[0] for row in await cursor.fetchall()]

                invalid_permissions = set(permissions) - set(valid_permissions)
                if invalid_permissions:
                    await cursor.close()
                    await cnx.close()
                    return sanic_json({
                        "status": "error",
                        "message": f"Invalid permissions: {', '.join(invalid_permissions)}"
                    }, status=400)

                for perm_name in permissions:
                    await cursor.execute("""
                        INSERT INTO role_permissions (role_id, permission_id)
                        SELECT %s, id FROM permissions WHERE name = %s
                    """, (role_id, perm_name))

        await cnx.commit()
        await cursor.close()
        await cnx.close()

        final_role_name = new_role_name if new_role_name else current_role_name

        return sanic_json({
            "status": "success",
            "message": f"Role '{final_role_name}' updated successfully.",
            "role_id": int(role_id),
            "role_name": final_role_name,
            "permissions_updated": permissions is not None
        })

    except Exception as e:
        return sanic_json({
            "status": "error",
            "message": f"Database error: {str(e)}"
        }, status=500)


@require_permission("manage_users") 
@app.route("/roles/<role_id>", methods=["GET"])
async def get_role_details(request, role_id):
    """Get detailed information about a specific role"""
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        await cursor.execute("""
            SELECT id, role_name, created_by, created_at, updated_by, updated_at 
            FROM roles WHERE id = %s
        """, (role_id,))
        role_row = await cursor.fetchone()

        if not role_row:
            await cursor.close()
            await cnx.close()
            return sanic_json({
                "status": "error",
                "message": "Role not found."
            }, status=404)

        role_id, role_name, created_by, created_at, updated_by, updated_at = role_row

        await cursor.execute("""
            SELECT p.id, p.name, p.description
            FROM role_permissions rp
            JOIN permissions p ON rp.permission_id = p.id
            WHERE rp.role_id = %s
            ORDER BY p.name
        """, (role_id,))
        permission_rows = await cursor.fetchall()

        permissions = [
            {
                "id": perm_id,
                "name": perm_name,
                "description": perm_desc or ""
            }
            for perm_id, perm_name, perm_desc in permission_rows
        ]

        await cursor.execute("""
            SELECT id, username FROM users WHERE role = %s ORDER BY username
        """, (role_name,))
        user_rows = await cursor.fetchall()

        users = [
            {"id": user_id, "username": username}
            for user_id, username in user_rows
        ]

        await cursor.close()
        await cnx.close()

        return sanic_json({
            "status": "success",
            "role": {
                "id": role_id,
                "role_name": role_name,
                "created_by": created_by,
                "created_at": created_at.strftime("%Y-%m-%d %H:%M:%S") if created_at else None,
                "updated_by": updated_by,
                "updated_at": updated_at.strftime("%Y-%m-%d %H:%M:%S") if updated_at else None,
                "permissions": permissions,
                "users": users,
                "user_count": len(users),
                "permission_count": len(permissions)
            }
        })

    except Exception as e:
        return sanic_json({
            "status": "error",
            "message": f"Database error: {str(e)}"
        }, status=500)

@require_permission("manage_users")
@app.route("/ldap", methods=["GET"])
async def get_ldap_config(request):
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        # Was `FROM ldap_config ORDER BY updated_at` — neither exists. The
        # table is `ldap_conf` (init_userdb.sql, and every other query in this
        # file) and it has created_at, not updated_at. It is also a
        # single-row table pinned to id=1 by the upsert below, so there is
        # nothing to order by.
        await cursor.execute("SELECT * FROM ldap_conf LIMIT 1")
        columns = [col[0] for col in cursor.description]
        row = await cursor.fetchone()
        await cursor.close(); await cnx.close()
        
        if row:
            config = dict(zip(columns, row))
            config["bind_password"] = "********" if config.get("bind_password") else ""
            return sanic_json({"status": "success", "config": config})
        else:
            return sanic_json({"status": "success", "config": None})
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

@require_permission("manage_users")
@app.route("/ldap", methods=["POST"])
async def upsert_ldap(request):
    data = request.json or {}

    required_fields = [
        "ldap_host", "ldap_port", "bind_dn", "bind_password",
        "users_base", "group_base", "login_filter"
    ]
    missing = [f for f in required_fields if not data.get(f)]
    if missing:
        return sanic_json({
            "status": "error",
            "message": f"Missing required fields: {', '.join(missing)}"
        }, status=400)

    try:
        ldap_port = int(data["ldap_port"])
    except ValueError:
        return sanic_json({"status": "error", "message": "ldap_port must be integer"}, status=400)

    encrypted_password = encrypt_password(data["bind_password"])

    sql = """
    INSERT INTO ldap_conf
        (id, ldap_host, ldap_port, bind_dn, bind_password, users_base, group_base, login_filter)
    VALUES
        (1, %s, %s, %s, %s, %s, %s, %s)
    ON DUPLICATE KEY UPDATE
        ldap_host = VALUES(ldap_host),
        ldap_port = VALUES(ldap_port),
        bind_dn = VALUES(bind_dn),
        bind_password = VALUES(bind_password),
        users_base = VALUES(users_base),
        group_base = VALUES(group_base),
        login_filter = VALUES(login_filter)
    """

    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        await cursor.execute(sql, (
            data["ldap_host"],
            ldap_port,
            data["bind_dn"],
            encrypted_password,
            data["users_base"],
            data["group_base"],
            data["login_filter"],
        ))
        await cnx.commit()
        await cursor.close(); await cnx.close()

        return sanic_json({
            "status": "success",
            "message": "LDAP configuration upserted (single-row table enforced)."
        })
    except Exception as e:
        return sanic_json({"status": "error", "message": f"Database error: {str(e)}"}, status=500)


@app.route("/ldap/groups", methods=["GET"])
@require_permission("manage_system")
async def get_ldap_groups(request):
    data = request.json or {}
     
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        await cursor.execute("""
            SELECT ldap_host, ldap_port, bind_dn, bind_password, group_base
            FROM ldap_conf
        """, )
        row = await cursor.fetchone()
        await cursor.close(); await cnx.close()

        if not row:
            return response.json({"status": "error", "message": "LDAP Conf. Error"}, status=404)

        ldap_host, ldap_port, bind_dn, encrypted_password, group_base = row
        bind_password = decrypt_password(encrypted_password)

        tls_config = Tls(validate=ssl.CERT_REQUIRED)

        server = Server(ldap_host, port=ldap_port, use_ssl=True, tls=tls_config, connect_timeout=3)
        conn = Connection(server, user=bind_dn, password=bind_password, auto_bind=True)
 
        conn.search(
            search_base=group_base,
            search_filter="(objectClass=groupOfNames)",
            attributes=["cn", "member"]
        )

        groups = [{
            "dn": entry.entry_dn,
            "cn": entry.cn.value,
            "members": entry.member.values if "member" in entry else []
        } for entry in conn.entries]

        conn.unbind()

        return response.json({"status": "success", "groups": groups})

    except Exception as e:
        print(f"[!] LDAP group listing error: {e}")
        return response.json({"status": "error", "message": "LDAP error"}, status=500)

@app.route("/change-password", methods=["POST"])
async def change_password(request):
    try:
        data_raw = request.json or {}
        pw_data = ChangePasswordRequest(**data_raw)
        username = pw_data.username
        current_password = pw_data.current_password
        new_password = pw_data.new_password
    except Exception as e:
        return sanic_json({"status": "error", "message": f"Invalid input: {str(e)}"}, status=400)

    # The username used to come from the request body, so anyone who knew
    # another account's current password could rotate it out from under them.
    # Self-service means self: bind to the session.
    session_user = current_user(request)
    if not session_user:
        return _unauthorized("Authentication required")
    if username != session_user["username"]:
        return sanic_json(
            {"status": "error", "message": "You can only change your own password."},
            status=403,
        )

    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        await cursor.execute("SELECT password FROM users WHERE username = %s LIMIT 1", (username,))
        row = await cursor.fetchone()
        if not row:
            await cursor.close(); await cnx.close()
            return sanic_json({"status": "error", "message": "No user found"}, status=404)

        db_password = row[0]
        if not bcrypt.checkpw(current_password.encode('utf-8'), db_password.encode('utf-8')):
            await cursor.close(); await cnx.close()
            return sanic_json({"status": "error", "message": "Current password is wrong"}, status=401)

        new_hashed = bcrypt.hashpw(new_password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')
        # Clearing the flag here is what lets the account back into the rest
        # of the console. Done in the same statement as the hash so a session
        # cannot end up with a new password and still be gated.
        await cursor.execute(
            "UPDATE users SET password = %s, must_change_password = 0 "
            "WHERE username = %s",
            (new_hashed, username))
        await cnx.commit()
        await cursor.close(); await cnx.close()

        # Includes this browser's own session: changing a password should end
        # every session that was issued against the old one.
        await revoke_user_sessions(int(session_user["user_id"]), "password changed")

        resp = sanic_json({
            "status": "success",
            "message": "Password changed. Please sign in again.",
            "reauth_required": True,
        })
        _clear_session_cookie(resp)
        return resp

    except Exception as e:
        return sanic_json({"status": "error", "message": f"Database Error: {e}"}, status=500)

@require_permission("manage_agent")
@app.route("/<agent>/restart", methods=["POST"])
async def trigger_restart(request, agent):
    """Restart the agent process."""
    body, status = await _agent_lifecycle_command(
        agent, "/restart", "restart", "Restart from UI",
        queue_action="restart_service")
    await _record_lifecycle(request, agent, "restart_service", body, status)
    return sanic_json(body, status=status)


@require_permission("manage_agent")
@app.route("/<agent>/reload_auth", methods=["POST"])
async def trigger_reload_auth(request, agent):
    """Make the agent re-fetch its key from /api/agents/bootstrap."""
    body, status = await _agent_lifecycle_command(
        agent, "/reload_auth", "reload_auth", "Auth reload from UI")
    await _record_lifecycle(request, agent, "reload_auth", body, status)
    return sanic_json(body, status=status)


@require_permission("manage_agent")
@app.route("/<agent>/self_destruct", methods=["POST"])
async def trigger_self_destruct(request, agent):
    """Uninstall the agent from its host. Irreversible.

    Goes to the agent's own `/self_destruct`, not `/soar/execute`: the latter
    accepts only the members of `ActionType`, which does not include this, so
    every uninstall push returned 501 for a command the agent implements.

    The agent removes its autostart synchronously and answers with the result,
    so a reply here means the watchdog is really gone - not merely that the
    request was delivered. See `Sentora/main.py:disable_autostart`.
    """
    # Remove the autostart from here first, before asking the agent to remove
    # itself.
    #
    # The agent gained a `disable_autostart` step that runs before it deletes
    # anything, and that is the right place for it - but it only exists in
    # builds that carry it. An older binary answers "Destruction initiated",
    # deletes its files, exits, and is restarted a minute later by the
    # scheduled task's watchdog. The server reads HTTP 200 and reports a
    # completed uninstall.
    #
    # This does not depend on the agent implementing uninstall correctly. The
    # server generated the installer, so it knows what the autostart is
    # called, and `run_cmd` is in every version of `ActionType` - so this
    # works against an agent of any age. Doing it first also means the
    # watchdog is already gone by the time the files go.
    autostart = await _remove_agent_autostart(agent)

    body, status = await _agent_lifecycle_command(
        agent, "/self_destruct", "self_destruct", "Self-destruct from UI",
        timeout=30)

    # One sentence, not two overlapping ones.
    #
    # The server removes the autostart and then the agent removes it again, so
    # both had something to say about it and the message concatenated them:
    # "Autostart removed (ERROR: cannot find the file) ... Autostart: removed
    # via schtasks". Both halves were true and together they read as a
    # failure. What the operator needs is whether the watchdog is gone.
    body["autostart_removal"] = autostart
    body["message"] = (
        f"Autostart: {autostart}. "
        + ("The agent is deleting itself and exiting."
           if status < 400 else
           f"The agent did not confirm the uninstall: {body.get('message', '')}")
    )
    await _record_lifecycle(request, agent, "self_destruct", body, status)
    return sanic_json(body, status=status)


async def _agent_os(agent: str) -> str:
    """Lower-cased `os_info` for an agent, or "" when it is not known."""
    try:
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute(
            "SELECT os_info FROM agent_info ORDER BY last_seen DESC LIMIT 1")
        row = await cur.fetchone()
        await cur.close()
        await cnx.close()
        return str((row or [""])[0] or "").lower()
    except Exception as e:
        print(f"[!] could not read os_info for {agent}: {e}", flush=True)
        return ""


async def _remove_agent_autostart(agent: str) -> str:
    """Delete whatever relaunches this agent, using a command it will accept.

    Returns a sentence describing what happened, for the operator and the
    action log. Never raises: this runs before the uninstall proper, and
    failing here must not stop that from being attempted.
    """
    os_info = await _agent_os(agent)
    if "windows" in os_info:
        argv = ["schtasks", "/Delete", "/TN", "SentoraAgent", "/F"]
    elif os_info:
        argv = ["systemctl", "disable", "--now", "sentora-agent"]
    else:
        return ("skipped - this agent's OS is unknown, so there was no safe "
                "command to send")

    try:
        resp = await call_agent_soar(
            agent, "run_cmd", argv,
            comment="Remove autostart before uninstall",
            background_queue=False)
    except Exception as e:
        return f"failed to send ({e})"

    if resp.get("ok"):
        return f"removed via {' '.join(argv)}"
    return (f"not confirmed ({resp.get('error') or resp.get('message')}) - "
            f"if the agent comes back, run `{' '.join(argv)}` on the host")


async def _record_lifecycle(request, agent: str, action: str, body: dict, status: int):
    """Write what happened to the agent's action log and the audit trail.

    Three outcomes, not two. `SUCCESS` used to be recorded whenever the call
    was delivered, so a push the agent answered 501 to was logged as a
    completed uninstall - and an operator reading the log later would have had
    no way to know the endpoint still had an agent on it.
    """
    outcome = {"success": "SUCCESS", "pending": "QUEUED"}.get(
        str(body.get("status")), "FAILED")
    detail = str(body.get("message") or "")[:400]
    try:
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute(
            "INSERT INTO soar_actions (action, target, status, comment) VALUES (%s, %s, %s, %s)",
            (action, "sentora-agent", outcome, detail))
        await cnx.commit()
        await cur.close(); await cnx.close()
    except Exception as e:
        print(f"[!] could not record {action} for {agent}: {e}", flush=True)
    await audit_log(request, action.upper(), agent, f"{outcome}: {detail}")


@require_permission("manage_agent")
@app.route("/devices/<agent>")
async def device_health(request, agent):
    try:
        cnx = await connect_db_for_agent(agent)
        cursor = await cnx.cursor()

        await cursor.execute("SELECT COUNT(*) FROM agent_info")
        row = await cursor.fetchone()

        await cursor.close()
        await cnx.close()

        exists = row[0] > 0
        status = "healthy" if exists else "unhealthy"

        return sanic_json({
            "agent": agent,
            "exists": exists,
            "status": status
        })
    
    except Exception as e:
        return sanic_json({
            "agent": agent,
            "status": "unreachable",
            "error": f"DB error: {str(e)}"
        }, status=500)

    except Exception as e:
        return sanic_json({
            "agent": agent,
            "status": "unreachable",
            "error": str(e)
        }, status=500)

@app.route("/<agent>/automations/pending", methods=["GET"], name="get_pending_auto_compat")
@app.route("/api/agents/<agent>/automations/pending", methods=["GET"], name="get_pending_auto_api")
async def get_pending_automations_for_agent(request, agent):
    """Endpoint for agent to pull pending tasks. Polled every few seconds by
    every active agent — must NEVER 500 on a fresh agent whose `automations`
    table hasn't been created yet (the schema is provisioned lazily). Without
    this, agents log a flood of `fetch_pending_tasks failed ... 500 ...`.
    """
    denied = await _require_agent(request, agent)
    if denied:
        return denied

    # `async with` rather than a close on the happy path. This endpoint is
    # polled by every agent every few seconds, so it was the fastest way to
    # drain the connection pool: any error between acquiring the connection
    # and the explicit close leaked it permanently, and once ten had leaked
    # every request in the server waited on an empty pool.
    try:
        async with agent_conn(agent) as cnx:
            cur = await cnx.cursor(dictionary=True)
            try:
                try:
                    await cur.execute(
                        "SELECT id as task_id, action as type, target as target, comment as params_comment FROM automations "
                        "WHERE status='pending' AND `timestamp` <= NOW()"
                    )
                except Exception as qerr:
                    err_str = str(qerr)
                    if "1146" in err_str or "doesn't exist" in err_str.lower():
                        return sanic_json({"status": "success", "tasks": []})
                    raise
                rows = await cur.fetchall()
            finally:
                await _close_async_quiet(cur)

        def _params_for(action: str, target_raw):
            params: dict = {"target": target_raw}
            act = (action or "").strip().lower()
            if act == "run_cmd":
                cmd_list = None
                if isinstance(target_raw, list):
                    cmd_list = [str(x) for x in target_raw if x not in (None, "")]
                elif isinstance(target_raw, (str, bytes)):
                    s = target_raw.decode() if isinstance(target_raw, bytes) else target_raw
                    s = (s or "").strip()
                    if s.startswith("[") and s.endswith("]"):
                        try:
                            parsed = pyjson.loads(s)
                            if isinstance(parsed, list):
                                cmd_list = [str(x) for x in parsed if x not in (None, "")]
                        except Exception:
                            cmd_list = None
                    if cmd_list is None and s:
                        import shlex
                        try:
                            cmd_list = shlex.split(s, posix=False)
                        except Exception:
                            cmd_list = [s]
                params["cmd"] = cmd_list or []
            return params

        tasks = [
            {
                "id": r["task_id"],
                "type": r["type"],
                "params": {
                    **_params_for(r["type"], r["target"]),
                    "comment": r["params_comment"],
                },
            }
            for r in rows
        ]
        return sanic_json({"status": "success", "tasks": tasks})
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

# A task may only move from outstanding to finished, never back, and never to
# a state that would hide it from the agent that still has to run it.
_TERMINAL_AUTOMATION_STATES = {"completed", "failed", "cancelled", "success"}


@app.route("/automations/<task_id:int>/report", methods=["POST"])
async def report_automation_result_by_id(request, task_id):
    """Endpoint for an agent to report that it finished a task.

    `agent` used to come from the request body, which made it a claim rather
    than an identity. It now comes from the key, and the update is scoped to
    that agent's own database.
    """
    data = request.json or {}
    status = str(data.get("status", "completed")).strip().lower()
    output = data.get("output", "")

    agent = await _validate_agent_auth(request)
    if agent is None or agent == "*":
        # "*" is the fleet-wide legacy secret and names no host, so it cannot
        # say which agent finished a task.
        return sanic_json({"status": "error", "message": "unauthorized"}, status=401)

    if status not in _TERMINAL_AUTOMATION_STATES:
        return sanic_json({
            "status": "error",
            "message": f"status must be one of {sorted(_TERMINAL_AUTOMATION_STATES)}",
        }, status=400)

    try:
        async with agent_conn(agent) as cnx:
            cur = await cnx.cursor()
            try:
                # Only a task that is still outstanding may be closed. Without
                # this, the same id can be reported repeatedly and a task that
                # already ran can be rewritten - and an attacker walking
                # task_id from 1 could mark the queue done before the agent
                # ever polled it.
                await cur.execute(
                    "UPDATE automations SET status=%s, "
                    "comment=CONCAT(COALESCE(comment,''), %s), updated_at=NOW() "
                    "WHERE id=%s AND status IN ('pending','active')",
                    (status, f" | Agent Report: {output}"[:200], task_id)
                )
                changed = cur.rowcount
                await cnx.commit()
            finally:
                await _close_async_quiet(cur)
        if not changed:
            return sanic_json({
                "status": "error",
                "message": "no outstanding task with that id for this agent",
            }, status=404)
        return sanic_json({"status": "success"})
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

@app.route("/db-status")
@require_permission("manage_db")
async def db_status(request):
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        await cursor.execute("SELECT VERSION();")
        results = await cursor.fetchall()
        version_str = results[0][0] if results else "Unknown"
        await cursor.close(); await cnx.close()
        return sanic_json({"mysql": f"online, version: {version_str}"})
    except Exception as e:
        return sanic_json({"error": f"Database connection failed: {e}"}, status=500)


def _agent_versions() -> dict:
    """agent_name -> the version it reported on its last handshake.

    Keyed under both spellings, for the same reason `_agent_display_names` is:
    callers arrive with the database-derived name, whose hyphens
    `server._sanitize_db_name` has already flattened. Keying only on the
    stored name would leave every hyphenated host looking un-versioned - which
    reads exactly like an agent too old to report one, and is the mistake this
    codebase has now made in five places.

    Empty on any failure. This runs on the agents list, and a fleet that
    cannot be listed because a version column is missing is a worse outcome
    than a fleet listed without versions.
    """
    try:
        with sync_mysql_conn("userdb") as conn:
            cur = conn.cursor()
            try:
                cur.execute("SELECT agent_name, agent_version "
                            "FROM agent_identities WHERE agent_version IS NOT NULL")
                versions: dict = {}
                for stored, reported in cur.fetchall():
                    hyphen, underscore, _ = _agent_name_forms(stored)
                    for form in (stored, hyphen, underscore):
                        if form:
                            versions[form] = reported
                return versions
            finally:
                cur.close()
    except Exception as e:
        print(f"[!] could not read agent versions: {e}", flush=True)
        return {}


def _agent_display_names() -> dict:
    """agent_name -> display_name, for the agents that have one.

    Missing column and missing table both come back empty. This runs on the
    agents list, which is the first page an operator opens; failing it over a
    cosmetic column would hide the entire estate.
    """
    try:
        with sync_mysql_conn("userdb") as conn:
            cur = conn.cursor()
            try:
                cur.execute(
                    "SELECT agent_name, display_name FROM agent_identities "
                    "WHERE display_name IS NOT NULL AND display_name <> ''")
                # Keyed under both spellings. Callers look these up by the
                # database-derived name, whose hyphens have been flattened to
                # underscores, so keying only on the stored name meant a
                # renamed host with a hyphen kept showing its hostname - the
                # rename appeared to do nothing. See `_get_agent_keys`.
                names: dict = {}
                for stored, display in cur.fetchall():
                    hyphen, underscore, _ = _agent_name_forms(stored)
                    for form in (stored, hyphen, underscore):
                        if form:
                            names[form] = display
                return names
            finally:
                cur.close()
    except Exception as e:
        print(f"[!] could not read agent display names: {e}", flush=True)
        return {}


@app.patch("/api/agents/<agent>/display-name")
@require_permission("manage_agent")
async def set_agent_display_name(request, agent):
    """Rename a device for display. The agent's identity does not change.

    `agent_name` is what the agent's database is named after
    (server._sanitize_db_name), what SOAR actions route by, and what the agent
    holds in its own config. Changing it would mean moving a database,
    re-issuing the agent's configuration, and reconciling every stored row
    that refers to the old name.

    What an operator actually wants is to see "Web server 1" instead of
    "ip-172-31-42-49", and that is what this does. Clearing it falls back to
    the real name rather than leaving a device with no label at all.
    """
    body = request.json or {}
    raw = body.get("display_name")
    name = (str(raw).strip() if raw is not None else "")

    if len(name) > 128:
        return sanic_json({"status": "error",
                           "message": "display name is limited to 128 characters"},
                          status=400)
    # Control characters would corrupt the console's own log lines, where
    # agent names are interpolated into text an operator reads.
    if any(ch in name for ch in ("\x00", "\n", "\r", "\t")):
        return sanic_json({"status": "error",
                           "message": "display name cannot contain control characters"},
                          status=400)

    # Both spellings - see `_get_agent_keys`. The console routes by the
    # database-derived name, which has had its hyphens flattened, so an exact
    # match here reported "no such agent" for every host with a hyphen in it.
    hyphen, underscore, _ = _agent_name_forms(agent)
    candidates = [n for n in dict.fromkeys([agent, hyphen, underscore]) if n]

    def _write():
        with sync_mysql_conn("userdb") as conn:
            cur = conn.cursor()
            try:
                placeholders = ",".join(["%s"] * len(candidates))
                cur.execute(
                    f"SELECT agent_name FROM agent_identities "
                    f"WHERE agent_name IN ({placeholders})", candidates)
                row = cur.fetchone()
                if not row:
                    return False
                cur.execute(
                    "UPDATE agent_identities SET display_name=%s WHERE agent_name=%s",
                    (name or None, row[0]))
                conn.commit()
                return True
            finally:
                cur.close()

    try:
        found = await asyncio.to_thread(_write)
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

    if not found:
        return sanic_json({"status": "error", "message": "unknown agent"}, status=404)

    await audit_log(request, "RENAME_AGENT", agent,
                    f"display name set to {name!r}" if name
                    else "display name cleared")
    return sanic_json({"status": "success", "agent": agent,
                       "display_name": name or None})

@require_permission("read_telemetry")
@app.route("/devices")
async def list_agents(request):
    try:
        def get_agent_names():
            with sync_mysql_conn() as conn:
                cursor = conn.cursor()
                try:
                    cursor.execute("SHOW DATABASES")
                    databases = cursor.fetchall()
                    return sorted([db[0].replace('_db', '') for db in databases if db[0].endswith('_db')])
                finally:
                    cursor.close()

        agents_names = await asyncio.to_thread(get_agent_names)

        async def get_agent_details(name):
            try:
                cnx = await connect_db_for_agent(name)
                cur = await cnx.cursor(dictionary=True)
                await cur.execute("SELECT public_ip, os_info, last_seen FROM agent_info ORDER BY last_seen DESC LIMIT 1")
                row = await cur.fetchone()
                await cur.close()
                await cnx.close()

                status = "Offline"
                last_seen_str = "Never"
                if row:
                    ls = row.get("last_seen")
                    if isinstance(ls, datetime):
                        last_seen_str = ls.strftime("%Y-%m-%d %H:%M:%S")
                        delta = abs((datetime.now() - ls).total_seconds())
                        if delta < 90:
                            status = "Online"
                    
                    return {
                        "name": name,
                        "status": status,
                        "last_seen": last_seen_str,
                        "public_ip": row.get("public_ip", "-"),
                        "os_info": row.get("os_info", "Generic Linux")
                    }
                return {"name": name, "status": "Offline", "last_seen": "Never", "public_ip": "-", "os_info": "Unknown"}
            except Exception as e:
                return {"name": name, "status": "Error", "last_seen": "DB Error", "public_ip": "-", "os_info": "Unknown"}

        agent_details = await asyncio.gather(*(get_agent_details(n) for n in agents_names))

        # One query for every agent rather than one per agent: this endpoint
        # already fans out a connection per host, and the names live together
        # in userdb.
        names = await asyncio.to_thread(_agent_display_names)
        versions = await asyncio.to_thread(_agent_versions)
        for detail in agent_details:
            detail["display_name"] = names.get(detail["name"]) or None
            reported = versions.get(detail["name"])
            state, why = product_version.agent_support(reported)
            detail["agent_version"] = reported
            detail["version_state"] = state
            detail["version_detail"] = why
            # Whether a command would reach it *right now*, which is not the
            # same question as whether it is reporting. Telemetry and control
            # travel on different connections, so an agent can be sending data
            # and be uncommandable - and the console has been unable to say
            # which since the fallback went.
            detail["channel_connected"] = _linked_agent(detail["name"]) is not None

        return sanic_json({"agents": agent_details,
                           "server_version": product_version.SERVER_VERSION,
                           "current_agent_version": product_version.CURRENT_AGENT_VERSION})
    except Exception as e:
        return sanic_json({"error": f"Database error while listing agents: {e}"}, status=500)


    
@require_permission("manage_db")
@app.route("/restart-db", methods=["POST"])
async def restart_db(request):
    try:
        # Off the event loop. `subprocess.run` blocks until docker finishes
        # restarting the container — several seconds during which this worker
        # served no other request, including the health check.
        result = await asyncio.to_thread(
            subprocess.run,
            ["docker", "compose", "restart", "mysql"],
            capture_output=True,
            text=True,
            timeout=120,
        )

        if result.returncode != 0:
            return sanic_json({
                "status": "error",
                "message": f"Restart failed: {result.stderr.strip()}"
            }, status=500)

        return sanic_json({
            "status": "success",
            "message": "MySQL container restarted using docker compose."
        })

    except Exception as e:
        return sanic_json({
            "status": "error",
            "message": f"Exception while restarting container: {str(e)}"
        }, status=500)



#: Tables this endpoint may empty. Both are deduplicated on ingest, which is
#: why `_clear_one_table` exists rather than a bare TRUNCATE.
CLEARABLE_TABLES = ["events_alert", "siem_events"]


async def _clear_one_table(cursor, table: str) -> None:
    """Empty a table *and* forget what it has already seen.

    The TRUNCATE alone left every ingest fingerprint behind, and that is not a
    tidiness problem - it permanently blinds the table.

    Ingest writes a fingerprint per row into `ingest_fingerprint` and skips any
    row whose fingerprint it already holds. Clearing the rows without clearing
    the fingerprints therefore means every future row of that table is skipped
    as a duplicate of a row that no longer exists. The table stays empty for
    ever, and the agent is told the server already has the data - so it marks
    those rows sent and never offers them again.

    Seen live: an agent shipped 507 new `siem_events` rows into a table
    holding zero, and every layer reported success. One click on Clear had
    turned the most important table on the console into a permanent silence
    that looks exactly like a quiet host.
    """
    await cursor.execute(f"TRUNCATE TABLE {table}")
    # Scoped to this table: the fingerprints for everything else describe rows
    # that are still there, and dropping them would re-admit every duplicate
    # the ingest path exists to reject.
    await cursor.execute(
        "DELETE FROM ingest_fingerprint WHERE table_name=%s", (table,))


@require_permission("clear_logs")
@app.route("/<agent>/clear/<table>", methods=["POST"])
async def clear_table(request, agent, table):
    if table not in CLEARABLE_TABLES:
        return sanic_json({"status": "error", "message": f"{table} Can't be cleaned."}, status=403)

    try:
        cnx = await connect_db_for_agent(agent)
        cursor = await cnx.cursor()
        await _clear_one_table(cursor, table)
        await cnx.commit()
        await cursor.close(); await cnx.close()
        return sanic_json({"status": "success", "message": f"{table} tablosu temizlendi."})
    except Exception as e:
        return sanic_json({"status": "error", "message": f"DB error: {e}"}, status=500)

@require_permission("clear_logs")
@app.route("/<agent>/clear_delayed/<table>", methods=["POST"])
async def clear_table_delayed(request, agent, table):
    data = request.json or {}
    delay = int(data.get("delay", 0))

    if table not in CLEARABLE_TABLES:
        return sanic_json({"status": "error", "message": f"{table} is not a table this endpoint may clear"}, status=403)

    if delay <= 0:
        return sanic_json({"status": "error", "message": "delay must be a positive number of seconds"}, status=400)

    async def delayed_clear():
        await asyncio.sleep(delay)
        try:
            cnx = await connect_db_for_agent(agent)
            cursor = await cnx.cursor()
            # The same clear as the immediate route, through the same
            # function. Two copies of a TRUNCATE is how one of them came to
            # forget the fingerprints in the first place.
            await _clear_one_table(cursor, table)
            await cnx.commit()
            await cursor.close(); await cnx.close()
            print(f"{table} {delay} cleaned after this.")
        except Exception as e:
            print(f"Error occurred: {e}")

    request.app.add_task(delayed_clear())  
    return sanic_json({
        "status": "scheduled",
        "message": f"{table} {delay} going to be clean after."
    })

@require_permission("read_telemetry")
@app.route("/all_alerts")
async def get_all_alerts(request):

    try:
        def _list_agents():
            with sync_mysql_conn() as conn:
                cursor = conn.cursor()
                try:
                    cursor.execute("SHOW DATABASES")
                    return [db[0].replace('_db', '') for db in cursor.fetchall() if db[0].endswith('_db')]
                finally:
                    cursor.close()
        agents = await asyncio.to_thread(_list_agents)
        per_agent = max(1, min(int(request.args.get("per_agent", 100)), 500))

        # `events_alert` stores source/message/severity encrypted at rest. The
        # per-agent endpoint runs them through stream_from_db_dec; this one did
        # not, so Global Alerts rendered raw `enc::gAAAAA...` ciphertext in the
        # Source and Message columns.
        key_b64 = getattr(app.ctx, "fernet_key", None)
        if not key_b64:
            key_b64 = bootstrap_client.get_fernet_key()
        fernet_obj = Fernet(key_b64.encode("utf-8") if isinstance(key_b64, str) else key_b64)
        encrypted_fields = ENCRYPTED_FIELDS_MAP["events_alert"]

        async def _for_agent(agent):
            async with agent_conn(agent) as cnx:
                cursor = await cnx.cursor()
                try:
                    await cursor.execute(
                        "SELECT * FROM events_alert ORDER BY id DESC LIMIT %s", (per_agent,)
                    )
                    columns = [col[0] for col in cursor.description]
                    rows = await cursor.fetchall()
                except Exception as e:
                    if "1146" in str(e) or "doesn't exist" in str(e).lower():
                        return []
                    raise
                finally:
                    await cursor.close()

            out = []
            for row in rows:
                alert = decrypt_row_fields(dict(zip(columns, row)), encrypted_fields, fernet_obj)
                alert['agent'] = agent
                out.append(alert)
            return out

        # Walked one agent at a time before, so page latency scaled with fleet
        # size even though the queries are independent.
        buckets = await gather_over_agents(_for_agent, agents)
        all_alerts = [a for bucket in buckets for a in bucket]

        json_stream = '[' + ','.join(
            [pyjson.dumps(alert, cls=CustomEncoder, ensure_ascii=False) for alert in all_alerts]
        ) + ']'

        return HTTPResponse(
            body=json_stream,
            content_type="application/json; charset=utf-8"
        )

    except Exception as e:
        return sanic_json({
            "status": "error",
            "message": f"Error while streaming alerts from all agents: {e}"
        }, status=500)


async def periodic_critical_alerts_check():
    while True:
        try:
            await asyncio.sleep(300)
            print("[+] Running periodic critical alerts check...")
            await asyncio.to_thread(dispatch_all_critical_alerts)
        except Exception as e:
            print(f"[!] Error in periodic critical alerts check: {e}")

async def periodic_soar_automation_check():
    while True:
        try:
            await asyncio.sleep(30)
            def _list_agents():
                with sync_mysql_conn() as c:
                    cur = c.cursor()
                    cur.execute("SHOW DATABASES")
                    agents = [db[0].replace('_db', '') for db in cur.fetchall() if db[0].endswith('_db')]
                    cur.close()
                    return agents
            agents = await asyncio.to_thread(_list_agents)

            for agent in agents:
                try:
                    await _run_due_automations_logic(agent)
                except Exception as e:
                    print(f"[SOAR] Periodic check failed for agent {agent}: {e}")
        except Exception as e:
            print(f"[SOAR] Global periodic check failed: {e}")

async def _run_due_automations_logic(agent):
    async with agent_conn(agent) as cnx:
        cur = await cnx.cursor(dictionary=True)
        try:
            await cur.execute(
                "SELECT * FROM automations WHERE status='pending' AND `timestamp` <= NOW() ORDER BY `timestamp` ASC"
            )
            rows = await cur.fetchall()
        finally:
            await cur.close()

    for rec in rows or []:
        async with agent_conn(agent) as cnx:
            cur = await cnx.cursor()
            try:
                await cur.execute("UPDATE automations SET status='active' WHERE id=%s", (rec["id"],))
                await cnx.commit()
            finally:
                await cur.close()

        result = await call_agent_soar(
            agent,
            action=rec["action"],
            target=rec["target"],
            comment=f"automation#{rec['id']} | {rec.get('comment') or ''}".strip(),
            event_id=rec.get("event_id"),
            # This row IS the queue entry being dispatched. call_agent_soar
            # defaults to background_queue=True, which inserts a fresh
            # 'pending' automation so an agent that missed the push can still
            # poll for it - correct when something new is being ordered, and a
            # loop when the dispatcher does it: every execution queued its own
            # successor, one row per cycle, forever.
            #
            # A manually triggered self_destruct therefore kept re-arming
            # itself and destroyed the agent again on every reinstall, with
            # the comment chain growing an `automation#N |` prefix each time.
            background_queue=False,
        )

        status = "completed" if result.get("ok") else "failed"
        async with agent_conn(agent) as cnx:
            cur = await cnx.cursor()
            try:
                await cur.execute("UPDATE automations SET status=%s WHERE id=%s", (status, rec["id"]))
                await cnx.commit()
            finally:
                await cur.close()

@app.before_server_start
async def setup_background_tasks(app):
    # Single argument: Sanic deprecated the `loop` parameter on listeners and
    # removes it in v26.6. Nothing here used it.
    worker_name = os.environ.get("SANIC_WORKER_NAME", "")
    if "0-0" in worker_name or not worker_name:
        print(f"[*] Starting background tasks in worker: {worker_name or 'single'}")
        app.add_task(periodic_critical_alerts_check())
        app.add_task(periodic_soar_automation_check())
        app.add_task(periodic_threat_intel_update())
        app.add_task(periodic_vuln_scan())
        app.add_task(periodic_session_purge())
    else:
        pass


async def periodic_session_purge():
    """Drop expired/revoked session rows hourly so the table stays bounded."""
    while True:
        await asyncio.sleep(3600)
        try:
            async with userdb_conn() as cnx:
                cur = await cnx.cursor()
                try:
                    removed = await session_store.purge_expired(cur)
                    await cnx.commit()
                finally:
                    await cur.close()
            if removed:
                print(f"[Session] Purged {removed} expired session row(s).")
        except Exception as e:
            print(f"[!] Session purge failed: {e}")


VULN_SCAN_INTERVAL = int(os.getenv("VULN_SCAN_INTERVAL", "1800"))


# Fan-out queries used to walk agent databases one at a time, so page latency
# was N x single-agent latency. Every one of them is independent — different
# databases, different pools — so they run concurrently now.
#
# Bounded because each slot holds a pooled connection, and an unbounded gather
# across a large fleet would open one per agent at once.
_AGENT_FANOUT_LIMIT = int(os.getenv("AGENT_FANOUT_LIMIT", "8"))


async def gather_over_agents(fn, agents: list, limit: int = None) -> list:
    """Run `fn(agent)` across agents concurrently; skip the ones that fail.

    A newly enrolled agent may not have provisioned a table yet, and one
    unreachable database must not take a whole page down with it — so
    exceptions are swallowed per agent and simply contribute nothing.
    """
    sem = asyncio.Semaphore(limit or _AGENT_FANOUT_LIMIT)

    async def _one(agent):
        async with sem:
            try:
                return await fn(agent)
            except Exception as e:
                print(f"[fanout] skip {agent}: {e}", flush=True)
                return None

    results = await asyncio.gather(*(_one(a) for a in agents))
    return [r for r in results if r is not None]


# asyncio keeps only a *weak* reference to a running task. A task nobody holds
# can be garbage collected mid-execution, so a fire-and-forget
# `asyncio.create_task(...)` silently may or may not finish — which for log
# indexing means audit entries that intermittently never reach OpenSearch, with
# nothing in the log to say so.
#
# Holding a strong reference until the task completes is the documented fix.
_background_tasks: set = set()


def _spawn(coro, *, label: str = "background"):
    """Fire-and-forget a coroutine without letting the GC eat it."""
    task = asyncio.create_task(coro)
    _background_tasks.add(task)
    task.add_done_callback(_background_tasks.discard)

    def _report(t: asyncio.Task):
        # An exception in a discarded task is otherwise only surfaced by
        # asyncio's "exception was never retrieved" warning at GC time.
        if not t.cancelled() and t.exception() is not None:
            print(f"[!] {label} task failed: {t.exception()}", flush=True)

    task.add_done_callback(_report)
    return task


async def _list_agent_names_sync() -> list:
    """Return all `*_db` agent names by querying SHOW DATABASES via the
    sync-root pool, mirroring the /devices endpoint."""
    def _q():
        with sync_mysql_conn() as conn:
            cur = conn.cursor()
            try:
                cur.execute("SHOW DATABASES")
                return [r[0].replace("_db", "") for r in cur.fetchall() if r[0].endswith("_db")]
            finally:
                cur.close()
    return await asyncio.to_thread(_q)


async def periodic_vuln_scan():
    """Periodic OSV vulnerability scan against every agent's `packages` table.
    Moved off the agent so endpoints don't burn CPU on a 5-minute find_vuln run.

    On first run, the OSV endpoint (public vs air-gap mirror) is resolved
    once — see modules/vuln_scanner.resolve_osv_endpoint for the selection
    logic. Subsequent cycles reuse the same URL.
    """
    await asyncio.sleep(60)
    from scanners.vuln import scan_all_agents, resolve_osv_endpoint
    await asyncio.to_thread(resolve_osv_endpoint)
    while True:
        try:
            key_b64 = getattr(app.ctx, "fernet_key", None) or bootstrap_client.get_fernet_key()
            print("[VulnScanner] starting cycle...", flush=True)
            stats = await scan_all_agents(key_b64, connect_db_for_agent, _list_agent_names_sync)
            total_inserted = sum(s.get("inserted", 0) for s in stats)
            print(f"[VulnScanner] cycle done — agents={len(stats)} new_findings={total_inserted}", flush=True)
        except Exception as e:
            print(f"[VulnScanner] cycle failed: {e}", flush=True)
        await asyncio.sleep(VULN_SCAN_INTERVAL)


@app.post("/<agent>/vulns/scan")
@require_permission("read_telemetry")
async def trigger_agent_vuln_scan(request, agent):
    """Manual trigger for a single-agent OSV scan."""
    try:
        from scanners import vuln as _vs
        if _vs._OSV_BASE is None:
            await asyncio.to_thread(_vs.resolve_osv_endpoint)
        scan_agent = _vs.scan_agent
        key_b64 = getattr(app.ctx, "fernet_key", None) or bootstrap_client.get_fernet_key()
        stat = await scan_agent(agent, key_b64, connect_db_for_agent)
        return sanic_json({"ok": True, **stat})
    except Exception as e:
        return sanic_json({"ok": False, "error": str(e)}, status=500)

MOCK_INDICATORS_TO_PURGE = [
    # Seeded by the previous mock updater. The empty-string SHA-256 was
    # labelled CRITICAL malware: matching against it would flag every empty
    # file on every endpoint. The two IPs were illustrative, not sourced.
    "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "185.220.101.5",
    "45.146.165.37",
]


async def purge_mock_threat_intel():
    """Delete the hardcoded indicators the old updater wrote.

    Removing them from the code is not enough — they are already in every
    deployment's database, and the empty-hash row is an active false-positive
    source. Matched on source too, so an operator who has legitimately added
    one of these addresses by hand keeps it.
    """
    try:
        with sync_mysql_conn("sentora_hub") as conn:
            cur = conn.cursor()
            try:
                placeholders = ",".join(["%s"] * len(MOCK_INDICATORS_TO_PURGE))
                cur.execute(
                    f"""DELETE FROM threat_intel
                         WHERE value IN ({placeholders})
                           AND source IN ('TorExitNode', 'BruteForceList', 'MalwareDB')""",
                    MOCK_INDICATORS_TO_PURGE,
                )
                removed = cur.rowcount
                conn.commit()
            finally:
                cur.close()
        if removed:
            print(f"[ThreatIntel] Purged {removed} mock indicator(s) from the previous seeder.")
    except Exception as e:
        print(f"[ThreatIntel] Could not purge mock indicators: {e}")


def _write_threat_indicators(indicators: list) -> int:
    """Upsert indicators and prune stale ones. Blocking; call via to_thread.

    Shared by the hourly refresher and the manual one behind
    POST /threat-intel/refresh, so the two cannot drift into writing different
    things.
    """
    if not indicators:
        return 0
    with sync_mysql_conn("sentora_hub") as conn:
        cur = conn.cursor()
        try:
            cur.executemany(
                """INSERT INTO threat_intel
                       (type, value, source, severity, description, last_seen)
                   VALUES (%s, %s, %s, %s, %s, NOW())
                   ON DUPLICATE KEY UPDATE
                       source = VALUES(source),
                       severity = VALUES(severity),
                       description = VALUES(description),
                       last_seen = NOW()""",
                [(i.type, i.value, i.source, i.severity, i.description) for i in indicators],
            )
            written = cur.rowcount or 0
            # Indicators that stopped appearing in the feeds. An address that
            # hosted a C2 last quarter usually belongs to someone else now,
            # and keeping it produces false positives indefinitely.
            cur.execute(
                "DELETE FROM threat_intel "
                "WHERE source LIKE 'abuse.ch/%%' "
                "AND last_seen < (NOW() - INTERVAL %s DAY)",
                (threat_feeds.STALE_AFTER_DAYS,),
            )
            conn.commit()
            return written
        finally:
            cur.close()


async def periodic_threat_intel_update():
    """Refresh indicators from real feeds, hourly.

    This used to insert three hardcoded rows. See core/threat_feeds.py for
    what replaced it and how to point it at an internal mirror.
    """
    await purge_mock_threat_intel()

    while True:
        try:
            if threat_feeds.mode() == "off":
                print("[ThreatIntel] Disabled (THREAT_INTEL_MODE=off).")
            else:
                indicators, errors = await asyncio.to_thread(threat_feeds.fetch_all)

                for err in errors:
                    print(f"[ThreatIntel] feed error — {err}")

                if indicators:
                    written = await asyncio.to_thread(_write_threat_indicators, indicators)
                    print(f"[ThreatIntel] {len(indicators)} indicator(s) from "
                          f"{len(threat_feeds.enabled_feeds())} feed(s); {written} row(s) written.")
                elif not errors:
                    print("[ThreatIntel] Feeds returned no indicators.")
        except Exception as e:
            print(f"[ThreatIntel] Refresh failed: {e}")

        await asyncio.sleep(3600)

@require_permission("read_telemetry")
@app.route("/threat-intel")
async def get_threat_intel(request):
    """Indicators, filtered, plus the stats that say whether the feeds work.

    Previously returned every row unfiltered — up to MAX_PER_FEED per feed —
    and nothing about freshness. "Is this data current?" is the first question
    an operator has about a threat feed, and until the mock seeder was removed
    the honest answer was no.
    """
    q = (request.args.get("q") or "").strip()
    ioc_type = (request.args.get("type") or "").strip()
    source = (request.args.get("source") or "").strip()
    limit = max(1, min(int(request.args.get("limit", 200) or 200), 1000))

    where = []
    params: list = []
    if q:
        where.append("(value LIKE %s OR description LIKE %s)")
        params.extend([f"%{q}%", f"%{q}%"])
    if ioc_type:
        where.append("type = %s")
        params.append(ioc_type)
    if source:
        where.append("source = %s")
        params.append(source)
    clause = f"WHERE {' AND '.join(where)}" if where else ""

    try:
        async with aiomysql.create_pool(
            host=DB_HOST, port=DB_PORT, user=DB_USER, password=DB_PASSWORD,
            db="sentora_hub", autocommit=True,
        ) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor(aiomysql.DictCursor) as cur:
                    await cur.execute(
                        f"SELECT * FROM threat_intel {clause} "
                        f"ORDER BY last_seen DESC, id DESC LIMIT %s",
                        (*params, limit),
                    )
                    rows = await cur.fetchall()

                    await cur.execute(f"SELECT COUNT(*) AS n FROM threat_intel {clause}", params)
                    matched = (await cur.fetchone() or {}).get("n", 0)

                    # Per-source counts and freshness. A source that has
                    # stopped refreshing shows an old last_seen here rather
                    # than just silently shrinking as pruning catches up.
                    await cur.execute(
                        "SELECT source, COUNT(*) AS n, MAX(last_seen) AS newest "
                        "FROM threat_intel GROUP BY source ORDER BY n DESC"
                    )
                    by_source = await cur.fetchall()

                    await cur.execute(
                        "SELECT type, COUNT(*) AS n FROM threat_intel GROUP BY type ORDER BY n DESC"
                    )
                    by_type = await cur.fetchall()

                    await cur.execute("SELECT COUNT(*) AS n, MAX(last_seen) AS newest FROM threat_intel")
                    totals = await cur.fetchone() or {}

        return sanic_json(
            {
                "status": "success",
                "indicators": rows,
                "matched": matched,
                "returned": len(rows),
                "stats": {
                    "total": totals.get("n", 0),
                    "newest": totals.get("newest"),
                    "by_source": by_source,
                    "by_type": by_type,
                    "mode": threat_feeds.mode(),
                    "feeds_enabled": threat_feeds.enabled_feeds(),
                    "stale_after_days": threat_feeds.STALE_AFTER_DAYS,
                },
            },
            # `last_seen` and `created_at` are datetimes; the default
            # serializer cannot encode them, which is what made this endpoint
            # 500 for every populated table.
            dumps=lambda obj: pyjson.dumps(obj, ensure_ascii=False, cls=CustomEncoder),
        )
    except Exception as e:
        # There was no handler at all, so failures fell through to the global
        # one and surfaced as "Contact administrator" with the cause hidden.
        print(f"[!] threat-intel fetch failed: {e}")
        return sanic_json({"status": "error", "message": str(e)}, status=500)

@app.post("/threat-intel/refresh")
@require_permission("manage_system")
async def refresh_threat_intel(request):
    """Pull the feeds now instead of waiting for the hourly cycle.

    Without this, confirming that a feed works — or that an abuse.ch auth key
    fixed a 403 — means restarting the container or waiting an hour and
    reading the log.
    """
    if threat_feeds.mode() == "off":
        return sanic_json({
            "status": "error",
            "message": "Feeds are disabled (THREAT_INTEL_MODE=off).",
        }, status=400)

    try:
        indicators, errors = await asyncio.to_thread(threat_feeds.fetch_all)
        written = await asyncio.to_thread(_write_threat_indicators, indicators)
        await audit_log(request, "THREAT_INTEL_REFRESH", "manual",
                        f"{len(indicators)} indicator(s), {len(errors)} feed error(s)")
        return sanic_json({
            "status": "success",
            "fetched": len(indicators),
            "written": written,
            # Surfaced rather than logged: a feed that needs an auth key is
            # something the operator has to act on, and they are standing in
            # front of this button.
            "errors": errors,
        })
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)


@app.get("/api/ai/triage-stats")
@require_permission("read_telemetry")
async def get_triage_stats(request):
    """What the pre-LLM funnel discarded, and how much it saved.

    A filter you cannot see is a blind spot. The severity gate drops events
    based on a label the agent's rule file assigned — not the model — so the
    number of events the model never got to judge has to be inspectable, not
    inferred from a quiet queue.

    Counters are per server process and reset on restart; they measure the
    current run, not lifetime totals.
    """
    try:
        from core import triage
        data = triage.config()

        # Every count below is read from the agent databases. Ingest runs in
        # server.py and this endpoint is served by app.py, so any counter held
        # in module state would report zero here regardless of what the funnel
        # actually did.
        agents = await _list_agent_names_sync()

        async def _for_agent(agent):
            async with agent_conn(agent) as cnx:
                cur = await cnx.cursor(dictionary=True)
                try:
                    await cur.execute(
                        "SELECT COUNT(*) AS distinct_events, "
                        "       COALESCE(SUM(occurrences), 0) AS total_events "
                        "  FROM ai_dedup"
                    )
                    row = await cur.fetchone() or {}
                except Exception:
                    return None      # table not provisioned on this agent yet

                try:
                    await cur.execute(
                        "SELECT severity, dropped FROM ai_triage_drops"
                    )
                    drops = await cur.fetchall() or []
                except Exception:
                    drops = []       # nothing has been dropped on this agent
                finally:
                    await cur.close()
            return {
                "agent": agent,
                "distinct_events": int(row.get("distinct_events") or 0),
                "total_events": int(row.get("total_events") or 0),
                "drops": {d["severity"]: int(d["dropped"]) for d in drops},
            }

        per_agent = await gather_over_agents(_for_agent, agents)
        distinct = sum(a["distinct_events"] for a in per_agent)
        total = sum(a["total_events"] for a in per_agent)

        by_severity: dict = {}
        for a in per_agent:
            for sev, n in a["drops"].items():
                by_severity[sev] = by_severity.get(sev, 0) + n

        data["severity_gate"] = {
            "dropped_by_severity": by_severity,
            "dropped_total": sum(by_severity.values()),
            # The events the model was never given a chance to judge. Read
            # this before raising AI_MIN_SEVERITY.
            "note": ("Severity is assigned by the agent's rule file, not the "
                     "model. These events were not analysed."),
        }

        data["dedup"] = {
            "distinct_events": distinct,
            "total_events": total,
            "inferences_avoided": max(0, total - distinct),
            # What fraction of ingested events did NOT need their own
            # inference. This is the funnel's actual payoff.
            "suppression_rate": round(1 - (distinct / total), 4) if total else 0.0,
            "by_agent": sorted(per_agent, key=lambda a: a["total_events"], reverse=True),
        }
        # A zero here is ambiguous on its own — it means either that nothing
        # fell below the floor, or that no telemetry has arrived at all. The
        # dedup totals disambiguate it, so they are stated together.
        if total == 0:
            data["severity_gate"]["note"] += " No events ingested yet."
        return sanic_json({"status": "success", **data})
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)


@app.get("/api/attack/coverage")
@require_permission("read_telemetry")
async def get_attack_coverage(request):
    """Which ATT&CK techniques this deployment has actually seen.

    The honest answer to "what am I not detecting". Two numbers that are
    easily confused and are not the same thing:

    - `covered`: techniques some installed Sigma rule can detect. This is
      capability, and it is what a coverage heatmap usually shows.
    - `observed`: techniques that have actually fired on this estate, with
      counts. This is history.

    A technique can be covered and never observed, which is the normal and
    healthy case. The dangerous cell is the one that is neither: nothing
    installed would have caught it, so its absence from the console says
    nothing at all.

    Techniques come from the Sigma rules' own `tags`, so there is no mapping
    table here to fall out of date.
    """
    agent = request.args.get("agent")

    try:
        observed = await asyncio.to_thread(_observed_techniques, agent)
    except Exception as e:
        print(f"[!] attack coverage query failed: {e}", flush=True)
        observed = {}

    covered = sorted(_sigma_technique_index())

    # Compared through the technique hierarchy, not as strings.
    #
    # This was `set(covered) - set(observed)` both ways, so a rule tagged
    # `T1003.001` and an event carrying `T1003` never matched each other: one
    # landed in "covered but never seen" and the other in "seen with nothing
    # covering it", and both were wrong. Those two lists are the only ones an
    # operator acts on.
    states = attack.classify(covered, observed)
    return sanic_json({
        "status": "success",
        "covered": covered,
        "observed": [{"technique": k, "events": v}
                     for k, v in sorted(observed.items(), key=lambda kv: -kv[1])],
        "covered_count": len(covered),
        "observed_count": len(observed),
        # Covered and never seen. Normal, and the healthy majority.
        "quiet": attack.unseen(covered, observed),
        # Seen with nothing addressing it at all.
        "uncovered_but_seen": states["none"],
        # Seen, with rules for *other* sub-techniques of the same parent and
        # not this one. Reported separately because it is the state most
        # easily mistaken for coverage - the parent cell is green while this
        # particular action would go unnoticed - and because the fix differs:
        # `none` needs a rule written, this usually needs one widened.
        "covered_only_by_a_sibling": states["sibling"],
        # Seen at sub-technique granularity, with a rule that claims the whole
        # parent.
        "covered_by_the_parent": states["parent"],
        # What the grid rolls up, so a cell can say "3 sub-techniques" rather
        # than implying the whole technique.
        "subtechniques": attack.rollup(covered),
    })


def _observed_techniques(agent: str | None = None) -> dict:
    """ATT&CK technique -> how many events carried it.

    Lifted out of the coverage handler so the fleet report can ask the
    same question. A second copy of this SQL would answer it differently
    the first time either was changed, and a coverage number that
    disagrees with itself is worse than one that is merely wrong.
    """
    seen = {}
    with sync_mysql_conn() as conn:
        cursor = conn.cursor()
        try:
            cursor.execute("SHOW DATABASES")
            dbs = [r[0] for r in cursor.fetchall() if r[0].endswith("_db")]
            if agent:
                wanted = _agent_name_forms(agent)[1] + "_db"
                dbs = [d for d in dbs if d == wanted]
            for db in dbs:
                try:
                    cursor.execute(
                        f"SELECT techniques, COUNT(*) FROM {_quote_identifier(db)}"
                        f".siem_events WHERE techniques IS NOT NULL "
                        f"AND techniques <> '' GROUP BY techniques"
                    )
                except Exception:
                    # An agent enrolled before the column existed. Not an
                    # error, and not a reason to fail the whole request.
                    continue
                for value, count in cursor.fetchall():
                    for tech in str(value).split(","):
                        tech = tech.strip().upper()
                        if tech:
                            seen[tech] = seen.get(tech, 0) + int(count)
        finally:
            cursor.close()
    return seen


#: What the server holds for a table, so the two halves can be compared.
def _server_table_counts(agent: str, tables) -> dict:
    """Row count and newest row per table, from this agent's database."""
    counts: dict[str, dict] = {}
    db = _agent_name_forms(agent)[1] + "_db"
    with sync_mysql_conn() as conn:
        cursor = conn.cursor()
        try:
            for table in tables:
                try:
                    cursor.execute(
                        f"SELECT COUNT(*) FROM {_quote_identifier(db)}"
                        f".{_quote_identifier(table)}")
                    counts[table] = {"rows": int(cursor.fetchone()[0] or 0)}
                except Exception:
                    # A table this server does not have. A real state - the
                    # agent may be newer - and not a reason to fail the rest.
                    counts[table] = {"rows": None}
        finally:
            cursor.close()
    return counts


def _agent_db_created_at(agent: str):
    """When this agent's database on the server was created, or None.

    The missing half of the "lost in transit" verdict. A table the agent has
    shipped and the server does not hold reads as "accepted and discarded",
    and that is one of two very different situations:

      the server took the rows and threw them away        - a real fault
      the server's database was recreated after they were sent

    The second is ordinary - a re-enrolment, a deleted agent, a fresh volume -
    and the rows are simply gone rather than being dropped on the floor now.
    Reporting it as the first sent somebody hunting a bug that is not there,
    twice in one day, so the timestamp is worth one query.
    """
    db = _agent_name_forms(agent)[1] + "_db"
    try:
        with sync_mysql_conn() as conn:
            cursor = conn.cursor()
            try:
                cursor.execute(
                    f"SELECT created_at FROM {_quote_identifier(db)}"
                    f".ingest_epoch WHERE id = 1")
                row = cursor.fetchone()
                return row[0] if row else None
            finally:
                cursor.close()
    except Exception:
        # No epoch table means a database older than the stamp. Unknown is a
        # fine answer; it just costs the sharper wording below.
        return None


def _shipped_before(last_sent_at, db_created_at) -> bool:
    """Whether this table was last shipped before the server's database existed.

    `last_sent_at` is a unix timestamp from the agent; `db_created_at` is a
    MySQL DATETIME. Both are naive UTC on every deployment this runs on, and a
    comparison that cannot be made confidently returns False - the sharper
    wording is a bonus, and claiming a reset that did not happen would be a
    worse error than the vague verdict it replaces.
    """
    if last_sent_at is None or db_created_at is None:
        return False
    try:
        created = db_created_at.timestamp()
    except AttributeError:
        return False
    return float(last_sent_at) < created


def _classify_link(agent_side: dict, server_rows, db_created_at=None) -> tuple[str, str]:
    """Where the chain is broken for one table, if it is.

    The whole point of this file. Each of these states looked identical from
    the console before - an empty table - and each needs a different person to
    do a different thing.
    """
    held = int(agent_side.get("held") or 0)
    unsent = int(agent_side.get("unsent") or 0)
    shipped = int(agent_side.get("shipped") or 0)
    error = agent_side.get("last_error")

    if server_rows is None:
        return ("no table", "this server has no such table; the agent may be "
                            "newer than it")
    if held == 0:
        return ("not collected",
                "the agent has never written a row of this locally - the "
                "collector is not running, failed, or does not apply to this host")
    if error:
        return ("send failing", f"the agent could not ship it: {error}")
    if shipped == 0 and unsent > 0:
        return ("queued", f"{unsent} row(s) collected and none shipped yet")
    if shipped > 0 and server_rows == 0:
        # Two very different situations look identical here, and calling both
        # of them "discarded" sent somebody hunting a bug that was not there.
        if _shipped_before(agent_side.get("last_sent_at"), db_created_at):
            return ("server reset",
                    f"the agent last shipped this before this server's "
                    f"database for it was created - the {shipped} row(s) went "
                    f"to a database that no longer exists, and rows already "
                    f"marked sent are re-offered once, bounded")
        # The state that was invisible. The agent's log says "sent (50 rows)"
        # every cycle and the server holds none of them.
        return ("lost in transit",
                f"the agent has shipped {shipped} row(s) and this server holds "
                f"none - accepted and discarded, not a quiet host")
    if server_rows == 0:
        return ("empty", "nothing collected and nothing expected")
    return ("flowing", f"{server_rows} row(s) here")


@app.route("/telemetry-health/<agent>")
@require_permission("read_telemetry")
async def get_telemetry_health(request, agent):
    """Does this agent's telemetry actually arrive? Per table, end to end.

    Three bugs in one day had the same shape: every layer reported success,
    the chain was broken anyway, and the only symptom was an empty table -
    indistinguishable from a host that genuinely has nothing to report.

    A collector raised on its first line and the four after it never ran. The
    server accepted a batch and discarded every row. And the agent marks rows
    sent as soon as `sendall` returns, because the ingest protocol has no
    reply - so it reported success in all three cases and was not wrong to.

    Neither half can tell a quiet host from a broken pipeline on its own. This
    puts the two numbers next to each other, which is all it ever needed.
    """
    link = _linked_agent(agent)
    if link is None:
        return sanic_json({"status": "error",
                           "message": _no_channel_message(agent)}, status=502)
    try:
        body, status = await link.request("GET", "/telemetry/health", timeout=15)
    except agent_link.LinkError as e:
        return sanic_json({"status": "error",
                           "message": f"the agent did not answer: {e}"}, status=502)
    if status == 501:
        return sanic_json({
            "status": "error",
            "message": "This agent predates telemetry health reporting. "
                       "Rebuild and reinstall it.",
        }, status=501)

    agent_tables = (body or {}).get("tables") or {}
    server = await asyncio.to_thread(
        _server_table_counts, agent, sorted(agent_tables))
    # One query, so "shipped to a database that no longer exists" can be told
    # apart from "shipped and discarded".
    db_created_at = await asyncio.to_thread(_agent_db_created_at, agent)

    report = []
    for table in sorted(agent_tables):
        side = agent_tables[table] or {}
        rows = server.get(table, {}).get("rows")
        state, detail = _classify_link(side, rows, db_created_at)
        report.append({
            "table": table,
            "state": state,
            "detail": detail,
            "agent_held": side.get("held"),
            "agent_unsent": side.get("unsent"),
            "agent_shipped": side.get("shipped"),
            "agent_last_sent_at": side.get("last_sent_at"),
            "agent_last_error": side.get("last_error"),
            "server_rows": rows,
        })

    # `server reset` is deliberately not counted as broken. The rows are gone
    # and nothing is wrong now - counting it would keep the banner red over a
    # condition that resolves itself on the next re-offer.
    broken = [r for r in report if r["state"] in
              ("lost in transit", "send failing", "not collected")]
    return sanic_json({
        "status": "success",
        "agent": agent,
        "tables": report,
        "broken_count": len(broken),
    })


# ---------------------------------------------------------------------------
# Second factor
# ---------------------------------------------------------------------------

async def _totp_tables(cur) -> None:
    """Create the three tables on first use.

    Alongside the rest of this file's guarded DDL rather than in
    init_userdb.sql, so a deployment that predates two-factor gets them
    without a migration step somebody has to remember to run.
    """
    for statement in (totp_store.DDL_SECRETS, totp_store.DDL_RECOVERY,
                      totp_store.DDL_PENDING):
        await cur.execute(statement)


async def _totp_secret_for(cur, user_id: int) -> tuple[str | None, bool]:
    """(secret, confirmed) for a user, decrypted, or (None, False).

    An unreadable secret is reported as absent rather than raised. The one
    situation that produces it is a Fernet key that has changed, and the
    honest consequence is that the user has to enrol again - not that they
    cannot log in at all.
    """
    await _totp_tables(cur)
    await cur.execute(
        "SELECT secret, confirmed_at FROM user_totp WHERE user_id = %s",
        (user_id,))
    row = await cur.fetchone()
    if not row:
        return None, False
    try:
        secret = fernet.decrypt(row[0].encode()).decode()
    except Exception:
        print(f"[2fa] the stored secret for user {user_id} cannot be "
              f"decrypted; they will have to enrol again", flush=True)
        return None, False
    return secret, row[1] is not None


async def _totp_required(cur, user_id: int) -> bool:
    """Whether this user has a *confirmed* second factor of any kind.

    Confirmed, not merely enrolled. A secret written when the QR code was
    displayed and never proved would lock out everybody who scanned it into an
    app they then deleted.

    A registered security key counts. Reading only the TOTP table would let an
    operator who enrolled a key and never set up an authenticator sign in with
    a password alone - the console would show a second factor on the account
    and not ask for one, which is the worst of both: the appearance of the
    control without the control.
    """
    _, confirmed = await _totp_secret_for(cur, user_id)
    if confirmed:
        return True
    return await _has_security_key(cur, user_id)


async def _has_security_key(cur, user_id: int) -> bool:
    """Any key, for any origin.

    Deliberately not scoped to the current relying party, unlike the list the
    console shows. Whether a login needs a second factor is a property of the
    account; whether *this* browser can offer a key is a property of the
    origin. Conflating them would mean an operator with a key registered at a
    hostname could skip the second factor entirely by opening the console at
    an address - which turns the origin binding, the whole point of WebAuthn,
    into a bypass.
    """
    try:
        await _webauthn_tables(cur)
        await cur.execute(
            "SELECT 1 FROM user_webauthn WHERE user_id = %s LIMIT 1", (user_id,))
        return (await cur.fetchone()) is not None
    except Exception:
        # A missing table on an older database is "no keys", not a failed
        # login. This runs on the password path, and raising here would lock
        # everybody out of a console whose 2FA they had never enabled.
        return False


async def _begin_second_factor(request, *, user_id: int, username: str,
                               auth_type: str) -> str:
    """Park a half-authenticated login and return its one-use token.

    Deliberately not a session. Issuing a real one and upgrading it later
    leaves a window in which one factor is a logged-in operator, and every
    bug in that window is a bypass.
    """
    raw = totp_store.new_pending_token()
    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            await _totp_tables(cur)
            await cur.execute(
                "INSERT INTO totp_pending (token_hash, user_id, username, "
                "auth_type, ip, expires_at) VALUES (%s, %s, %s, %s, %s, "
                "DATE_ADD(NOW(), INTERVAL %s SECOND))",
                (totp_store.hash_pending(raw), user_id, username, auth_type,
                 _client_ip(request), totp_store.PENDING_TTL_SECONDS))
            # Opportunistic, and cheap: without it this table only grows, and
            # an abandoned login is exactly the row nobody comes back for.
            await cur.execute("DELETE FROM totp_pending WHERE expires_at < NOW()")
            await cnx.commit()
        finally:
            await cur.close()
    return raw


@app.post("/login/2fa")
async def complete_second_factor(request):
    """Finish a login that passed its password.

    Public in the session sense and not in any other: the pending token is a
    credential minted moments ago by a correct password, and it can do exactly
    one thing.
    """
    data = request.json or {}
    raw_token = (data.get("token") or "").strip()
    code = (data.get("code") or "").strip()
    ip = _client_ip(request)

    if not raw_token or not code:
        return response.json({"status": "error",
                              "message": "Both the login token and a code are "
                                         "required."}, status=400)

    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            await _totp_tables(cur)
            await cur.execute(
                "SELECT user_id, username, auth_type, attempts FROM "
                "totp_pending WHERE token_hash = %s AND expires_at > NOW()",
                (totp_store.hash_pending(raw_token),))
            pending = await cur.fetchone()
            if not pending:
                # Expired, already spent, or never existed - one message for
                # all three, because telling them apart tells an attacker
                # which half of a captured pair is still good.
                return response.json({
                    "status": "error",
                    "message": "This login has expired. Sign in again.",
                }, status=401)

            user_id, username, auth_type, attempts = pending
            if attempts >= totp_store.MAX_PENDING_ATTEMPTS:
                await cur.execute("DELETE FROM totp_pending WHERE token_hash = %s",
                                  (totp_store.hash_pending(raw_token),))
                await cnx.commit()
                await log_login_attempt(username, auth_type, "failure",
                                        "too many second-factor attempts", ip)
                return response.json({
                    "status": "error",
                    "message": "Too many attempts. Sign in again.",
                }, status=429)

            secret, confirmed = await _totp_secret_for(cur, user_id)
            if not secret or not confirmed:
                return response.json({
                    "status": "error",
                    "message": "This account has no second factor configured.",
                }, status=409)

            step = totp_store.verify(secret, code)
            used_recovery = False
            if step is None:
                # A recovery code, maybe. Checked second so a normal login
                # never touches the recovery table.
                await cur.execute(
                    "UPDATE user_recovery_codes SET used_at = NOW() "
                    " WHERE user_id = %s AND code_hash = %s AND used_at IS NULL",
                    (user_id, totp_store.hash_recovery(code)))
                used_recovery = cur.rowcount == 1
                if not used_recovery:
                    await cur.execute(
                        "UPDATE totp_pending SET attempts = attempts + 1 "
                        " WHERE token_hash = %s",
                        (totp_store.hash_pending(raw_token),))
                    await cnx.commit()
                    await log_login_attempt(username, auth_type, "failure",
                                            "second factor rejected", ip)
                    return response.json({
                        "status": "error",
                        "message": "That code is not valid.",
                    }, status=401)
            else:
                # Single use, and this is the line that makes it so.
                #
                # TOTP is a shared secret and a clock: the same six digits are
                # valid for the whole step, so without this anyone who watches
                # a code being typed has thirty seconds to use it themselves.
                # The comparison is `>=` rather than `=` so a code from
                # *earlier* in the accepted window cannot be replayed either.
                await cur.execute(
                    "UPDATE user_totp SET last_step = %s "
                    " WHERE user_id = %s AND (last_step IS NULL OR last_step < %s)",
                    (step, user_id, step))
                if cur.rowcount == 0:
                    await cur.execute(
                        "UPDATE totp_pending SET attempts = attempts + 1 "
                        " WHERE token_hash = %s",
                        (totp_store.hash_pending(raw_token),))
                    await cnx.commit()
                    await log_login_attempt(username, auth_type, "failure",
                                            "second-factor code reused", ip)
                    await _platform_event(
                        "TOTP_CODE_REPLAYED", subject=username, source_ip=ip,
                        detail="a code already used for this account was "
                               "presented again")
                    return response.json({
                        "status": "error",
                        "message": "That code has already been used. Wait for "
                                   "the next one.",
                    }, status=401)

            # Spent, whichever factor it was.
            await cur.execute("DELETE FROM totp_pending WHERE token_hash = %s",
                              (totp_store.hash_pending(raw_token),))
            await cur.execute(
                "SELECT role, must_change_password, created_at FROM users "
                " WHERE id = %s", (user_id,))
            role_row = await cur.fetchone()
            await cnx.commit()
        finally:
            await cur.close()

    return await _finish_second_factor_login(
        request, user_id=user_id, username=username, auth_type=auth_type,
        role_row=role_row, ip=ip,
        factor="recovery code" if used_recovery else "second factor",
        used_recovery=used_recovery)


async def _finish_second_factor_login(request, *, user_id: int, username: str,
                                      auth_type: str, role_row, ip: str,
                                      factor: str, used_recovery: bool = False):
    """Issue the session, once both factors are in.

    Shared with the security-key route rather than copied into it. Session
    issuance is the step where a mistake is a bypass, and two of them drift:
    the code that files a login and the code that scores one had already come
    apart once in this codebase, and nobody could see it because each side had
    its own idea of what counted.
    """
    role = role_row[0] if role_row else None
    must_change = bool(role_row[1]) if role_row else False

    try:
        user_perms = await get_user_permissions(user_id)
        raw_session = await _issue_session(
            request, user_id=user_id, username=username, role=role,
            auth_type=auth_type)
    except Exception as e:
        print(f"[!] {username} passed both factors but the session could not "
              f"be created: {e}", flush=True)
        return response.json({
            "status": "error",
            "message": "Both factors were accepted, but this server could not "
                       "start a session. This is a server fault, not a "
                       "credential one.",
        }, status=503)

    await log_login_attempt(username, auth_type, "success", factor, ip)
    if used_recovery:
        # Worth its own line and its own event: a recovery code means the
        # normal factor was unavailable, which is either an honest lost phone
        # or somebody who has the password and a stolen list.
        await _platform_event("RECOVERY_CODE_USED", subject=username,
                              source_ip=ip,
                              detail="signed in with a recovery code rather "
                                     "than an authenticator")

    resp = response.json({
        "status": "success",
        "message": "Login successful.",
        "user": {
            "id": user_id, "username": username, "auth_type": auth_type,
            "role": role, "permissions": user_perms,
            "must_change_password": must_change,
            "used_recovery_code": used_recovery,
        },
    })
    _set_session_cookie(resp, raw_session)
    return resp


@app.get("/api/2fa/status")
async def second_factor_status(request):
    """Whether the signed-in operator has a second factor, and how many
    recovery codes are left."""
    user_id = current_user_id(request)
    if not user_id:
        return sanic_json({"status": "error", "message": "Unauthorized"},
                          status=401)
    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            _, confirmed = await _totp_secret_for(cur, user_id)
            await cur.execute(
                "SELECT COUNT(*) FROM user_recovery_codes "
                " WHERE user_id = %s AND used_at IS NULL", (user_id,))
            remaining = (await cur.fetchone())[0]
        finally:
            await cur.close()
    return sanic_json({"status": "success", "enabled": confirmed,
                       "recovery_codes_remaining": int(remaining or 0)})


@app.post("/api/2fa/enrol")
async def second_factor_enrol(request):
    """Start enrolment: a new secret and the URI to scan.

    Nothing is enforced by this. The secret is written unconfirmed, and only
    `/api/2fa/confirm` - which requires a working code from it - turns it on.
    Enrolling in one step would lock out anybody whose app did not keep it.
    """
    user_id = current_user_id(request)
    if not user_id:
        return sanic_json({"status": "error", "message": "Unauthorized"},
                          status=401)

    secret = totp_store.new_secret()
    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            await _totp_tables(cur)
            await cur.execute("SELECT username FROM users WHERE id = %s",
                              (user_id,))
            row = await cur.fetchone()
            username = row[0] if row else str(user_id)
            # Replaces any unconfirmed attempt. Re-scanning a QR code that was
            # abandoned halfway is the ordinary case, not an error.
            await cur.execute(
                "INSERT INTO user_totp (user_id, secret, confirmed_at) "
                "VALUES (%s, %s, NULL) ON DUPLICATE KEY UPDATE "
                "secret = IF(confirmed_at IS NULL, VALUES(secret), secret)",
                (user_id, fernet.encrypt(secret.encode()).decode()))
            await cur.execute(
                "SELECT confirmed_at FROM user_totp WHERE user_id = %s",
                (user_id,))
            already = (await cur.fetchone())[0] is not None
            await cnx.commit()
        finally:
            await cur.close()

    if already:
        # Re-enrolling with one confirmed already would silently invalidate
        # the factor they are using. Disabling is a separate, deliberate act.
        return sanic_json({
            "status": "error",
            "message": "Two-factor is already enabled for this account. "
                       "Disable it first if you want to enrol a new device.",
        }, status=409)

    return sanic_json({
        "status": "success",
        "secret": secret,
        "uri": totp_store.provisioning_uri(secret, account=username,
                                           issuer="Sentora"),
        "digits": totp_store.DIGITS,
        "period": totp_store.STEP_SECONDS,
    })


@app.post("/api/2fa/confirm")
async def second_factor_confirm(request):
    """Prove a code from the new secret, and receive the recovery codes.

    The recovery codes are returned here and nowhere else, ever. They are
    stored as hashes, so the server genuinely cannot show them again - which
    is the property that makes them safe to store at all.
    """
    user_id = current_user_id(request)
    if not user_id:
        return sanic_json({"status": "error", "message": "Unauthorized"},
                          status=401)
    code = ((request.json or {}).get("code") or "").strip()

    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            secret, confirmed = await _totp_secret_for(cur, user_id)
            if not secret:
                return sanic_json({"status": "error",
                                   "message": "Start enrolment first."},
                                  status=409)
            if confirmed:
                return sanic_json({"status": "error",
                                   "message": "Two-factor is already enabled."},
                                  status=409)
            step = totp_store.verify(secret, code)
            if step is None:
                return sanic_json({
                    "status": "error",
                    "message": "That code is not valid. Check the time on the "
                               "device running your authenticator.",
                }, status=400)

            codes = totp_store.new_recovery_codes()
            await cur.execute(
                "UPDATE user_totp SET confirmed_at = NOW(), last_step = %s "
                " WHERE user_id = %s", (step, user_id))
            await cur.execute("DELETE FROM user_recovery_codes WHERE user_id = %s",
                              (user_id,))
            for one in codes:
                await cur.execute(
                    "INSERT INTO user_recovery_codes (user_id, code_hash) "
                    "VALUES (%s, %s)", (user_id, totp_store.hash_recovery(one)))
            await cnx.commit()
        finally:
            await cur.close()

    await audit_log(request, "2FA_ENABLED", str(user_id),
                    "second factor confirmed and recovery codes issued")
    return sanic_json({
        "status": "success",
        "recovery_codes": codes,
        "message": "Two-factor is on. Save these recovery codes now - they "
                   "are stored as hashes and cannot be shown again.",
    })


@app.post("/api/2fa/disable")
async def second_factor_disable(request):
    """Turn it off, with the current password.

    The password is required because a hijacked session must not be able to
    remove the control that would have stopped it. That is the whole reason
    this endpoint is not simply "delete the row".
    """
    user_id = current_user_id(request)
    if not user_id:
        return sanic_json({"status": "error", "message": "Unauthorized"},
                          status=401)
    password = ((request.json or {}).get("password") or "")
    if not password:
        return sanic_json({"status": "error",
                           "message": "Your password is required to turn off "
                                      "two-factor."}, status=400)

    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            await cur.execute("SELECT password FROM users WHERE id = %s",
                              (user_id,))
            row = await cur.fetchone()
            ok = False
            if row:
                try:
                    ok = bcrypt.checkpw(password.encode("utf-8"),
                                        row[0].encode("utf-8"))
                except ValueError:
                    ok = False       # an LDAP placeholder hash
            if not ok:
                await _platform_event(
                    "TOTP_DISABLE_REFUSED", subject=str(user_id),
                    source_ip=_client_ip(request),
                    detail="wrong password given when turning off two-factor")
                return sanic_json({"status": "error",
                                   "message": "That password is not correct."},
                                  status=403)
            await cur.execute("DELETE FROM user_totp WHERE user_id = %s",
                              (user_id,))
            await cur.execute("DELETE FROM user_recovery_codes WHERE user_id = %s",
                              (user_id,))
            await cnx.commit()
        finally:
            await cur.close()

    await audit_log(request, "2FA_DISABLED", str(user_id),
                    "second factor removed by the account holder")
    return sanic_json({"status": "success",
                       "message": "Two-factor is off for this account."})


# ---------------------------------------------------------------------------
# Security keys
#
# The second factor TOTP cannot be: a code typed into a convincing copy of this
# login page works on the real one, and the operator has no way to tell. A
# security key signs over the origin it is actually talking to, so it produces
# nothing usable on a copy.
#
# There is no configuration. The relying-party ID is derived from the request,
# which is why this works on a laptop at `localhost` and on an estate with a
# real hostname without either being told anything - and why some origins
# cannot host a key at all. `security.webauthn.usability` answers that, and the
# console shows the reason instead of a button that fails.
# ---------------------------------------------------------------------------

def _webauthn_lib():
    """The library, or None.

    Imported here rather than at module scope on purpose. This is an optional
    factor, and a deployment that pulls new code without reinstalling its
    dependencies should lose the security-key option, not the whole console.
    """
    try:
        import webauthn as _lib
        from webauthn.helpers import base64url_to_bytes, bytes_to_base64url
        from webauthn.helpers.structs import (
            AuthenticatorSelectionCriteria, PublicKeyCredentialDescriptor,
            ResidentKeyRequirement, UserVerificationRequirement,
        )
    except ImportError:
        return None
    return {
        "lib": _lib, "to_bytes": base64url_to_bytes, "to_b64": bytes_to_base64url,
        "Selection": AuthenticatorSelectionCriteria,
        "Descriptor": PublicKeyCredentialDescriptor,
        "ResidentKey": ResidentKeyRequirement,
        "UserVerification": UserVerificationRequirement,
    }


async def _webauthn_tables(cur) -> None:
    for statement in (webauthn_store.DDL_CREDENTIALS, webauthn_store.DDL_CHALLENGES):
        await cur.execute(statement)


def _webauthn_origin(request):
    """(rp_id, origin, reason) for the request in hand."""
    scheme = (request.scheme or "http").lower()
    try:
        rp_id, origin = webauthn_store.relying_party(request.host, scheme)
    except webauthn_store.Unusable as e:
        return None, None, str(e)
    return rp_id, origin, None


async def _webauthn_credentials(cur, user_id: int, rp_id: str) -> list:
    """This user's keys *for this origin*.

    Scoped to the relying party deliberately. A key registered while the
    console was at `localhost` is one the browser will not offer at a hostname,
    so listing it there shows an operator a factor they cannot use and then
    times out at the prompt with no explanation.
    """
    await cur.execute(
        "SELECT id, credential_id, public_key, sign_count, transports, "
        "       nickname, created_at, last_used_at "
        "  FROM user_webauthn WHERE user_id = %s AND rp_id = %s "
        " ORDER BY created_at",
        (user_id, rp_id))
    rows = await cur.fetchall()
    return [{"id": r[0], "credential_id": r[1], "public_key": r[2],
             "sign_count": int(r[3] or 0), "transports": r[4],
             "nickname": r[5], "created_at": r[6], "last_used_at": r[7]}
            for r in rows]


@app.get("/api/2fa/webauthn/capability")
async def webauthn_capability(request):
    """Whether a key can be used from where this browser is standing.

    The console asks before it offers anything, because the alternative is a
    button that produces a browser error naming neither the cause nor the fix.
    """
    user_id = current_user_id(request)
    if not user_id:
        return sanic_json({"status": "error", "message": "Unauthorized"}, status=401)

    if _webauthn_lib() is None:
        return sanic_json({
            "status": "success", "available": False,
            "reason": "This server was installed without the `webauthn` "
                      "package, so security keys are unavailable. One-time "
                      "codes are unaffected.",
            "credentials": [],
        })

    rp_id, _origin, reason = _webauthn_origin(request)
    if reason:
        return sanic_json({"status": "success", "available": False,
                           "reason": reason, "credentials": []})

    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            await _webauthn_tables(cur)
            creds = await _webauthn_credentials(cur, user_id, rp_id)
        finally:
            await cur.close()

    return sanic_json({
        "status": "success", "available": True, "reason": "", "rp_id": rp_id,
        "credentials": [{
            "id": c["id"],
            "label": webauthn_store.describe(c),
            "created_at": str(c["created_at"] or ""),
            "last_used_at": str(c["last_used_at"] or "") or None,
        } for c in creds],
    })


@app.post("/api/2fa/webauthn/register/begin")
async def webauthn_register_begin(request):
    """Options for creating a credential. Nothing becomes a factor yet."""
    user_id = current_user_id(request)
    if not user_id:
        return sanic_json({"status": "error", "message": "Unauthorized"}, status=401)

    lib = _webauthn_lib()
    if lib is None:
        return sanic_json({"status": "error",
                           "message": "This server has no WebAuthn support "
                                      "installed."}, status=501)

    rp_id, origin, reason = _webauthn_origin(request)
    if reason:
        return sanic_json({"status": "error", "message": reason}, status=409)

    challenge = webauthn_store.new_challenge()
    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            await _webauthn_tables(cur)
            await cur.execute("SELECT username FROM users WHERE id = %s", (user_id,))
            row = await cur.fetchone()
            username = row[0] if row else str(user_id)
            existing = await _webauthn_credentials(cur, user_id, rp_id)
            await cur.execute(
                "INSERT INTO webauthn_challenges (challenge, user_id, purpose, "
                "rp_id, origin, expires_at) VALUES (%s, %s, 'register', %s, %s, "
                "DATE_ADD(NOW(), INTERVAL %s SECOND))",
                (challenge, user_id, rp_id, origin,
                 webauthn_store.CHALLENGE_TTL_SECONDS))
            # Cheap, and this table only grows otherwise: an abandoned ceremony
            # is exactly the row nobody comes back for.
            await cur.execute("DELETE FROM webauthn_challenges WHERE expires_at < NOW()")
            await cnx.commit()
        finally:
            await cur.close()

    options = lib["lib"].generate_registration_options(
        rp_id=rp_id,
        rp_name=webauthn_store.RP_NAME,
        user_id=str(user_id).encode(),
        user_name=username,
        challenge=lib["to_bytes"](challenge),
        # Already-registered keys are excluded so the authenticator says "you
        # have one of these already" rather than quietly making a second
        # credential the operator cannot tell apart from the first.
        exclude_credentials=[
            lib["Descriptor"](id=lib["to_bytes"](c["credential_id"]))
            for c in existing
        ],
        authenticator_selection=lib["Selection"](
            resident_key=lib["ResidentKey"].PREFERRED,
            user_verification=lib["UserVerification"].PREFERRED,
        ),
    )
    return sanic_json({"status": "success",
                       "options": pyjson.loads(lib["lib"].options_to_json(options))})


@app.post("/api/2fa/webauthn/register/finish")
async def webauthn_register_finish(request):
    """Verify the attestation and store the credential."""
    user_id = current_user_id(request)
    if not user_id:
        return sanic_json({"status": "error", "message": "Unauthorized"}, status=401)

    lib = _webauthn_lib()
    if lib is None:
        return sanic_json({"status": "error",
                           "message": "This server has no WebAuthn support "
                                      "installed."}, status=501)

    data = request.json or {}
    credential = data.get("credential")
    nickname = (data.get("nickname") or "").strip()[:64] or None
    if not credential:
        return sanic_json({"status": "error",
                           "message": "No credential was sent."}, status=400)

    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            await _webauthn_tables(cur)
            # The challenge, the RP ID and the origin all come from the row we
            # wrote, never from the response. Verifying a response against the
            # origin the response itself claims is not a check.
            await cur.execute(
                "SELECT challenge, rp_id, origin FROM webauthn_challenges "
                " WHERE user_id = %s AND purpose = 'register' "
                "   AND expires_at > NOW() ORDER BY created_at DESC LIMIT 1",
                (user_id,))
            row = await cur.fetchone()
            if not row:
                return sanic_json({
                    "status": "error",
                    "message": "That registration expired. Start again.",
                }, status=400)
            challenge, rp_id, origin = row

            try:
                verified = lib["lib"].verify_registration_response(
                    credential=credential,
                    expected_challenge=lib["to_bytes"](challenge),
                    expected_rp_id=rp_id,
                    expected_origin=origin,
                )
            except Exception as e:
                await cur.execute(
                    "DELETE FROM webauthn_challenges WHERE challenge = %s",
                    (challenge,))
                await cnx.commit()
                return sanic_json({
                    "status": "error",
                    "message": f"That key could not be verified: {e}",
                }, status=400)

            # Single use, verified or not.
            await cur.execute("DELETE FROM webauthn_challenges WHERE challenge = %s",
                              (challenge,))
            transports = ""
            if isinstance(credential, dict):
                transports = ",".join(
                    str(t) for t in (credential.get("transports") or [])
                )[:128]
            await cur.execute(
                "INSERT INTO user_webauthn (user_id, credential_id, public_key, "
                "rp_id, sign_count, transports, nickname) "
                "VALUES (%s, %s, %s, %s, %s, %s, %s) "
                "ON DUPLICATE KEY UPDATE sign_count = VALUES(sign_count)",
                (user_id, lib["to_b64"](verified.credential_id),
                 lib["to_b64"](verified.credential_public_key), rp_id,
                 int(verified.sign_count or 0), transports, nickname))
            await cnx.commit()
        finally:
            await cur.close()

    await audit_log(request, "WEBAUTHN_KEY_ADDED", str(user_id),
                    f"security key registered for {rp_id}")
    return sanic_json({"status": "success",
                       "message": "Security key registered."})


@app.delete("/api/2fa/webauthn/<credential_row_id:int>")
async def webauthn_remove(request, credential_row_id: int):
    """Remove one key. Scoped to the caller, so an id from elsewhere does
    nothing rather than removing somebody else's factor."""
    user_id = current_user_id(request)
    if not user_id:
        return sanic_json({"status": "error", "message": "Unauthorized"}, status=401)

    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            await _webauthn_tables(cur)
            await cur.execute(
                "DELETE FROM user_webauthn WHERE id = %s AND user_id = %s",
                (credential_row_id, user_id))
            removed = cur.rowcount
            await cnx.commit()
        finally:
            await cur.close()

    if not removed:
        return sanic_json({"status": "error",
                           "message": "No such key on this account."}, status=404)
    await audit_log(request, "WEBAUTHN_KEY_REMOVED", str(user_id),
                    "security key removed by the account holder")
    return sanic_json({"status": "success", "message": "Security key removed."})


@app.post("/login/2fa/webauthn/begin")
async def webauthn_login_begin(request):
    """Options for asserting a key against a pending login.

    The pending token is the same one the code path uses: passing the password
    gets a short-lived credential that can do exactly one thing, and which
    factor finishes it is the operator's choice.
    """
    lib = _webauthn_lib()
    if lib is None:
        return sanic_json({"status": "error",
                           "message": "This server has no WebAuthn support "
                                      "installed."}, status=501)

    raw_token = ((request.json or {}).get("token") or "").strip()
    if not raw_token:
        return sanic_json({"status": "error",
                           "message": "The login token is required."}, status=400)

    rp_id, origin, reason = _webauthn_origin(request)
    if reason:
        return sanic_json({"status": "error", "message": reason}, status=409)

    challenge = webauthn_store.new_challenge()
    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            await _totp_tables(cur)
            await _webauthn_tables(cur)
            await cur.execute(
                "SELECT user_id FROM totp_pending WHERE token_hash = %s "
                "  AND expires_at > NOW()",
                (totp_store.hash_pending(raw_token),))
            row = await cur.fetchone()
            if not row:
                return sanic_json({
                    "status": "error",
                    "message": "This login has expired. Sign in again.",
                }, status=401)
            user_id = row[0]

            creds = await _webauthn_credentials(cur, user_id, rp_id)
            if not creds:
                # Said plainly rather than offered as an empty prompt. A key
                # registered at another origin is invisible to the browser
                # here, and "nothing happened" is the least useful answer.
                return sanic_json({
                    "status": "error",
                    "message": "No security key is registered for this account "
                               "at this address. Use a one-time code.",
                }, status=404)

            await cur.execute(
                "INSERT INTO webauthn_challenges (challenge, user_id, purpose, "
                "rp_id, origin, expires_at) VALUES (%s, %s, 'login', %s, %s, "
                "DATE_ADD(NOW(), INTERVAL %s SECOND))",
                (challenge, user_id, rp_id, origin,
                 webauthn_store.CHALLENGE_TTL_SECONDS))
            await cur.execute("DELETE FROM webauthn_challenges WHERE expires_at < NOW()")
            await cnx.commit()
        finally:
            await cur.close()

    options = lib["lib"].generate_authentication_options(
        rp_id=rp_id,
        challenge=lib["to_bytes"](challenge),
        allow_credentials=[
            lib["Descriptor"](id=lib["to_bytes"](c["credential_id"]))
            for c in creds
        ],
        user_verification=lib["UserVerification"].PREFERRED,
    )
    return sanic_json({"status": "success",
                       "options": pyjson.loads(lib["lib"].options_to_json(options))})


@app.post("/login/2fa/webauthn/finish")
async def webauthn_login_finish(request):
    """Verify the assertion and issue the session."""
    lib = _webauthn_lib()
    if lib is None:
        return sanic_json({"status": "error",
                           "message": "This server has no WebAuthn support "
                                      "installed."}, status=501)

    data = request.json or {}
    raw_token = (data.get("token") or "").strip()
    credential = data.get("credential")
    ip = _client_ip(request)
    if not raw_token or not credential:
        return sanic_json({"status": "error",
                           "message": "Both the login token and a credential "
                                      "are required."}, status=400)

    async with userdb_conn() as cnx:
        cur = await cnx.cursor()
        try:
            await _totp_tables(cur)
            await _webauthn_tables(cur)
            await cur.execute(
                "SELECT user_id, username, auth_type FROM totp_pending "
                " WHERE token_hash = %s AND expires_at > NOW()",
                (totp_store.hash_pending(raw_token),))
            pending = await cur.fetchone()
            if not pending:
                return sanic_json({
                    "status": "error",
                    "message": "This login has expired. Sign in again.",
                }, status=401)
            user_id, username, auth_type = pending

            await cur.execute(
                "SELECT challenge, rp_id, origin FROM webauthn_challenges "
                " WHERE user_id = %s AND purpose = 'login' "
                "   AND expires_at > NOW() ORDER BY created_at DESC LIMIT 1",
                (user_id,))
            row = await cur.fetchone()
            if not row:
                return sanic_json({
                    "status": "error",
                    "message": "That challenge expired. Try again.",
                }, status=400)
            challenge, rp_id, origin = row

            raw_id = credential.get("id") if isinstance(credential, dict) else None
            await cur.execute(
                "SELECT id, public_key, sign_count FROM user_webauthn "
                " WHERE user_id = %s AND rp_id = %s AND credential_id = %s",
                (user_id, rp_id, raw_id))
            stored = await cur.fetchone()
            if not stored:
                return sanic_json({
                    "status": "error",
                    "message": "That key is not registered for this account "
                               "at this address.",
                }, status=401)
            row_id, public_key, stored_count = stored

            try:
                verified = lib["lib"].verify_authentication_response(
                    credential=credential,
                    expected_challenge=lib["to_bytes"](challenge),
                    expected_rp_id=rp_id,
                    expected_origin=origin,
                    credential_public_key=lib["to_bytes"](public_key),
                    credential_current_sign_count=int(stored_count or 0),
                )
            except Exception as e:
                await cur.execute(
                    "DELETE FROM webauthn_challenges WHERE challenge = %s",
                    (challenge,))
                await cnx.commit()
                await log_login_attempt(username, auth_type, "failure",
                                        "security key rejected", ip)
                return sanic_json({
                    "status": "error",
                    "message": f"That key could not be verified: {e}",
                }, status=401)

            # Single use.
            await cur.execute("DELETE FROM webauthn_challenges WHERE challenge = %s",
                              (challenge,))

            presented = int(verified.new_sign_count or 0)
            if webauthn_store.counter_regressed(int(stored_count or 0), presented):
                # The one thing the counter is for. A key that has gone
                # backwards is a copy of one that has been used more times than
                # the copy knows about.
                await cur.execute("DELETE FROM totp_pending WHERE token_hash = %s",
                                  (totp_store.hash_pending(raw_token),))
                await cnx.commit()
                await log_login_attempt(username, auth_type, "failure",
                                        "security key counter went backwards", ip)
                await _platform_event(
                    "WEBAUTHN_COUNTER_REGRESSED", subject=username, source_ip=ip,
                    detail=f"a security key presented signature counter "
                           f"{presented} after {stored_count}; the key has "
                           f"either been cloned or the response replayed")
                return sanic_json({
                    "status": "error",
                    "message": "That key reported a signature counter lower "
                               "than the last one seen, which means it has "
                               "been copied. It has not been accepted.",
                }, status=401)

            await cur.execute(
                "UPDATE user_webauthn SET sign_count = %s, last_used_at = NOW() "
                " WHERE id = %s", (presented, row_id))
            await cur.execute("DELETE FROM totp_pending WHERE token_hash = %s",
                              (totp_store.hash_pending(raw_token),))
            await cur.execute(
                "SELECT role, must_change_password, created_at FROM users "
                " WHERE id = %s", (user_id,))
            role_row = await cur.fetchone()
            await cnx.commit()
        finally:
            await cur.close()

    return await _finish_second_factor_login(
        request, user_id=user_id, username=username, auth_type=auth_type,
        role_row=role_row, ip=ip, factor="security key")


@app.route("/api/platform/events")
@require_permission("manage_system")
async def get_platform_events(request):
    """Attacks on this platform, as opposed to attacks on the fleet.

    Every one of these was already detected. A lockout went to a container
    log, a rejected agent key went to a container log, a permission denial
    went to a table nobody opens unless they are already investigating - so
    the machine that watches every host in the fleet was the one nobody
    watched.

    Folded rows, not raw attempts: a brute-force is one row whose
    `occurrences` climbs, because a view that files four hundred rows for one
    attack is the thing that hides it.
    """
    hours = _positive_int(request.args.get("hours"), 24, 24 * 30)

    def _read():
        with sync_mysql_conn("userdb") as conn:
            cursor = conn.cursor(dictionary=True)
            try:
                cursor.execute(self_defence.DDL)
                cursor.execute(
                    "SELECT kind, severity, subject, source_ip, detail, "
                    "       occurrences, first_seen, last_seen "
                    "  FROM platform_events "
                    " WHERE last_seen > (NOW() - INTERVAL %s HOUR) "
                    " ORDER BY last_seen DESC LIMIT 500", (hours,))
                return cursor.fetchall() or []
            finally:
                cursor.close()

    try:
        rows = await asyncio.to_thread(_read)
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

    for row in rows:
        # What the kind means, sent with it. An operator seeing
        # FLEET_KEY_ON_CHANNEL for the first time should not have to go
        # looking for what it implies.
        row["explanation"] = self_defence.describe(row["kind"])[1]
        for field in ("first_seen", "last_seen"):
            if row.get(field) is not None:
                row[field] = row[field].isoformat()

    return sanic_json({
        "status": "success",
        "window_hours": hours,
        "events": rows,
        "summary": self_defence.summarise(rows),
        # Not an attack, and it belongs here anyway: this is the view that
        # answers "is the platform itself healthy", and history nobody can
        # decrypt is a fact about the platform rather than about any host.
        "decryption": decryption_health(),
    })


def _threat_trend(days: int = 14) -> list:
    """Detections per day across the fleet, in two counts that differ.

    `detections` is every event that fired a rule. `distinct` is how many
    *different* techniques were behind them, which is the number worth
    watching: a thousand detections from one noisy rule and a thousand from
    forty different techniques are the same bar and completely different days.

    Plotted together on two axes because the shape of one against the other is
    the signal - a spike in detections with a flat distinct line is almost
    always a rule that needs tuning, not an incident.
    """
    buckets: dict[str, dict] = {}
    with sync_mysql_conn() as conn:
        cursor = conn.cursor()
        try:
            cursor.execute("SHOW DATABASES")
            dbs = [r[0] for r in cursor.fetchall() if r[0].endswith("_db")]
            for db in dbs:
                try:
                    cursor.execute(
                        f"SELECT DATE(created_at) AS d, techniques, COUNT(*) "
                        f"FROM {_quote_identifier(db)}.siem_events "
                        f"WHERE created_at >= CURDATE() - INTERVAL %s DAY "
                        f"AND techniques IS NOT NULL AND techniques <> '' "
                        f"GROUP BY d, techniques",
                        (int(days),),
                    )
                except Exception:
                    # An agent enrolled before the column existed. Not an
                    # error, and not a reason to fail the whole chart.
                    continue
                for day, value, count in cursor.fetchall():
                    key = day.isoformat() if hasattr(day, "isoformat") else str(day)
                    bucket = buckets.setdefault(
                        key, {"date": key, "detections": 0, "techniques": set()})
                    bucket["detections"] += int(count)
                    for raw in str(value).split(","):
                        technique = attack.normalise(raw)
                        if technique:
                            bucket["techniques"].add(technique)
        finally:
            cursor.close()

    # Every day in the window, including the quiet ones. A line drawn only
    # through the days that had events invents a slope between them and makes
    # a two-day gap look like a gradual decline.
    import datetime
    today = datetime.date.today()
    series = []
    for offset in range(days, -1, -1):
        key = (today - datetime.timedelta(days=offset)).isoformat()
        bucket = buckets.get(key)
        series.append({
            "date": key,
            "detections": bucket["detections"] if bucket else 0,
            "distinct": len(bucket["techniques"]) if bucket else 0,
        })
    return series


@app.route("/threat-trend")
@require_permission("read_telemetry")
async def get_threat_trend(request):
    """Detections per day across the fleet, for the dashboard."""
    try:
        days = max(1, min(int(request.args.get("days", 14)), 90))
    except ValueError:
        days = 14
    try:
        series = await asyncio.to_thread(_threat_trend, days)
    except Exception as e:
        print(f"[!] threat trend query failed: {e}", flush=True)
        return sanic_json({"status": "error", "message": str(e)}, status=500)
    return sanic_json({"status": "success", "days": days, "series": series})


def _observed_chain(agent: str, hours: int = 168) -> list:
    """The tactics seen on one host, in kill-chain order, with when.

    A list of techniques tells you what happened. It does not tell you that
    execution was followed by persistence and then credential access, which is
    the difference between a suspicious command and an intrusion - and the
    difference an analyst is trying to establish when they open a host.

    Ordered by the chain rather than by the clock. Timestamps are kept per
    stage so the two can be read together: a chain whose stages are days apart
    is a different story from one that completed inside a minute, and only the
    order makes either legible.
    """
    stages: dict[str, dict] = {}
    db = _agent_name_forms(agent)[1] + "_db"
    with sync_mysql_conn() as conn:
        cursor = conn.cursor()
        try:
            cursor.execute(
                f"SELECT techniques, MIN(created_at), MAX(created_at), COUNT(*) "
                f"FROM {_quote_identifier(db)}.siem_events "
                f"WHERE techniques IS NOT NULL AND techniques <> '' "
                f"AND created_at >= NOW() - INTERVAL %s HOUR "
                f"GROUP BY techniques",
                (int(hours),),
            )
            rows = cursor.fetchall()
        except Exception as e:
            # An agent with no telemetry, or one enrolled before the column
            # existed. Not an error, and an empty chain is a true answer.
            print(f"[attack-chain] {agent}: {e}", flush=True)
            return []
        finally:
            cursor.close()

    tactics_for = _sigma_tactics_by_technique()
    for value, first, last, count in rows:
        for raw in str(value).split(","):
            technique = attack.normalise(raw)
            if not technique:
                continue
            # A technique with no tactic still happened. It lands at the end
            # of the chain rather than being dropped.
            for tactic in tactics_for.get(technique) or ["other"]:
                stage = stages.setdefault(tactic, {
                    "tactic": tactic, "techniques": set(),
                    "events": 0, "first_seen": first, "last_seen": last,
                })
                stage["techniques"].add(technique)
                stage["events"] += int(count)
                if first and (not stage["first_seen"] or first < stage["first_seen"]):
                    stage["first_seen"] = first
                if last and (not stage["last_seen"] or last > stage["last_seen"]):
                    stage["last_seen"] = last

    ordered = sorted(stages.values(),
                     key=lambda s: attack.tactic_position(s["tactic"]))
    return [{
        "tactic": s["tactic"],
        "position": attack.tactic_position(s["tactic"]),
        "techniques": sorted(s["techniques"]),
        "events": s["events"],
        "first_seen": s["first_seen"].isoformat() if s["first_seen"] else None,
        "last_seen": s["last_seen"].isoformat() if s["last_seen"] else None,
    } for s in ordered]


def _sigma_tactics_by_technique() -> dict:
    """Technique -> the tactics its rules are tagged with.

    From the rules themselves, so there is no mapping table here to drift out
    of date - the same reason the coverage index reads them rather than
    keeping a list.
    """
    mapping: dict[str, list] = {}
    try:
        from core.sigma_loader import load_dir
        path = os.getenv(
            "SIGMA_RULES_PATH",
            str(pathlib.Path(__file__).resolve().parent /
                "Sentora" / "conf" / "sigma"))
        for rule in load_dir(path).rules:
            for technique in rule.techniques:
                clean = attack.normalise(technique)
                if not clean:
                    continue
                known = mapping.setdefault(clean, [])
                for tactic in rule.tactics:
                    if tactic not in known:
                        known.append(tactic)
    except Exception as e:
        print(f"[attack-chain] could not read rule tactics: {e}", flush=True)
    return mapping


@app.route("/attack-chain/<agent>")
@require_permission("read_telemetry")
async def get_attack_chain(request, agent):
    """What happened on this host, laid out as a kill chain."""
    try:
        hours = max(1, min(int(request.args.get("hours", 168)), 24 * 90))
    except ValueError:
        hours = 168
    try:
        chain = await asyncio.to_thread(_observed_chain, agent, hours)
    except Exception as e:
        print(f"[!] attack chain query failed for {agent}: {e}", flush=True)
        return sanic_json({"status": "error", "message": str(e)}, status=500)

    return sanic_json({
        "status": "success",
        "agent": agent,
        "hours": hours,
        "chain": chain,
        # The full order, so the UI can draw the stages that did *not* happen
        # as gaps rather than omitting them. A chain with holes in it is the
        # readable thing; a list of four tactics is not.
        "tactic_order": list(attack.TACTIC_ORDER),
    })


def _sigma_technique_index() -> set:
    """Techniques the agent can detect: Sigma rules plus correlation.

    Read from the same directory and the same rule set the agent loads, so the
    console cannot claim coverage the endpoints do not have.

    Correlation is included because it is real coverage the operator has, and
    leaving it out would report T1110.003 as a blind spot on an estate that
    detects it. The two are counted together on purpose - the page's question
    is "would anything catch this", not "which layer would".
    """
    techniques = set()
    try:
        from core.sigma_loader import load_dir
        path = os.getenv("SIGMA_RULES_PATH",
                         os.path.join("Sentora", "conf", "sigma"))
        techniques |= load_dir(path).techniques
    except Exception as e:
        print(f"[!] could not read Sigma rules for coverage: {e}", flush=True)

    try:
        from core.correlation import techniques_covered
        techniques |= techniques_covered()
    except Exception as e:
        print(f"[!] could not read correlation rules for coverage: {e}", flush=True)

    return techniques




# ---------------------------------------------------------------------------
# Reports
# ---------------------------------------------------------------------------

def _report_rows(agent: str, table: str, limit: int = 500) -> list:
    """Rows from one agent table, decrypted, for a report.

    Uses `core.telemetry_crypto` rather than a second decrypt implementation.
    The one this file used to carry was case-sensitive about the `enc::`
    prefix and silently returned ciphertext - which is how the vulnerability
    scanner asked OSV about a package named `enc::gAAAA...` for a year.
    """
    from core import telemetry_crypto

    rows = _fetch_rows_safely(
        _agent_db_name(agent),
        f"SELECT * FROM {_quote_identifier(table)} ORDER BY id DESC LIMIT %s",
        (limit,), f"{table} for {agent}")
    return [telemetry_crypto.decrypt_item(table, dict(r)) for r in (rows or [])]


async def _gather_agent_report(agent: str) -> dict:
    """Everything one host's report needs, and what could not be read."""
    from core import telemetry_crypto

    info_rows = _fetch_rows_safely(
        _agent_db_name(agent),
        "SELECT public_ip, os_info, last_seen, hostname, mac_address "
        "FROM agent_info ORDER BY last_seen DESC LIMIT 1",
        (), f"agent_info for {agent}")
    info = dict(info_rows[0]) if info_rows else {}

    siem = await asyncio.to_thread(_report_rows, agent, "siem_events")
    alerts = await asyncio.to_thread(_report_rows, agent, "events_alert")
    vulns = await asyncio.to_thread(_report_rows, agent, "vulnerabilities_report")
    packages = await asyncio.to_thread(_report_rows, agent, "packages")
    insights = await asyncio.to_thread(_report_rows, agent, "ai_analysis_results")

    names = await asyncio.to_thread(_agent_display_names)

    # An agent can look healthy and have sent nothing: the address, hostname
    # and MAC arrive in the connection header, which needs no database on the
    # endpoint. Saying so is the difference between "quiet host" and "broken
    # collection", which look identical on paper.
    no_telemetry = not any((siem, alerts, packages, insights))

    note = None
    if packages:
        encrypted = sum(1 for p in packages
                        if str(p.get("package", "")).lower().startswith("enc::"))
        if encrypted == len(packages):
            note = (f"all {len(packages)} package names are still encrypted - "
                    f"this server's key does not match the agent's")

    return {
        "display_name": names.get(agent),
        "info": info,
        "siem_events": siem,
        "alerts": alerts,
        "vulnerabilities": vulns,
        "package_count": len(packages),
        "ai_insights": insights,
        "no_telemetry": no_telemetry,
        "vuln_scan_note": note,
    }


def _pdf_response(pdf: bytes, filename: str):
    return raw(pdf, content_type="application/pdf", headers={
        # `attachment` rather than inline: a PDF rendered in the browser from
        # the API origin would run in that origin's context.
        "Content-Disposition": f'attachment; filename="{filename}"',
        "Cache-Control": "no-store",
    })


@app.get("/api/agents/<agent>/report.pdf")
@require_permission("read_telemetry")
async def agent_report_pdf(request, agent):
    """One host, as a PDF."""
    from core.reporting import build_agent_report, filename_for, render_pdf

    try:
        data = await _gather_agent_report(agent)
        report = build_agent_report(agent, data)
        pdf = await asyncio.to_thread(render_pdf, report)
    except Exception as e:
        print(f"[!] agent report failed for {agent}: {e}", flush=True)
        return sanic_json({"status": "error", "message": str(e)}, status=500)

    await audit_log(request, "EXPORT_REPORT", agent,
                    f"agent PDF, {len(report.caveats)} caveat(s)")
    return _pdf_response(pdf, filename_for(report))


@app.get("/api/reports/fleet.pdf")
@require_permission("read_telemetry")
async def fleet_report_pdf(request):
    """The estate, as a PDF."""
    from core.reporting import build_fleet_report, filename_for, render_pdf

    try:
        listing = await list_agents(request)
        agents = (listing.raw_body and pyjson.loads(listing.body).get("agents")) or []
    except Exception as e:
        print(f"[!] fleet report could not list agents: {e}", flush=True)
        agents = []

    silent, totals, findings = [], {}, []
    for entry in agents:
        name = entry.get("name")
        if not name:
            continue
        try:
            vulns = await asyncio.to_thread(_report_rows, name,
                                            "vulnerabilities_report", 2000)
            alerts = await asyncio.to_thread(_report_rows, name, "events_alert", 200)
            siem = await asyncio.to_thread(_report_rows, name, "siem_events", 1)
            totals[name] = len(vulns)
            if not siem and not alerts:
                silent.append(name)
            for a in alerts:
                if str(a.get("source", "")).startswith("Correlation/"):
                    findings.append({**a, "agent": name})
        except Exception as e:
            print(f"[!] fleet report: {name} skipped ({e})", flush=True)

    observed = await asyncio.to_thread(_observed_techniques)

    report = build_fleet_report({
        "agents": agents,
        "silent_agents": silent,
        "covered_techniques": sorted(_sigma_technique_index()),
        "observed_techniques": sorted(observed),
        "correlation_findings": findings,
        "vulnerability_totals": totals,
    })
    try:
        pdf = await asyncio.to_thread(render_pdf, report)
    except Exception as e:
        print(f"[!] fleet report render failed: {e}", flush=True)
        return sanic_json({"status": "error", "message": str(e)}, status=500)

    await audit_log(request, "EXPORT_REPORT", "fleet",
                    f"{len(agents)} agent(s), {len(report.caveats)} caveat(s)")
    return _pdf_response(pdf, filename_for(report))

@app.get("/api/dashboard/summary")
@require_permission("read_telemetry")
async def get_dashboard_summary(request):
    """Everything the dashboard renders, in one round trip.

    The page used to issue seven requests, four of which walked every agent
    database — and then used most of the results only to count them. It pulled
    up to 100 fully decrypted alerts per agent to render two numbers.

    Counting in SQL instead means the payload no longer scales with fleet size.
    The alert *rows* are still available from /all_alerts for the page that
    actually lists them.
    """
    try:
        agents = await _list_agent_names_sync()

        async def _for_agent(agent):
            async with agent_conn(agent) as cnx:
                cur = await cnx.cursor(dictionary=True)
                try:
                    stats = {"agent": agent, "alerts": 0, "critical": 0,
                             "insights_1h": 0, "vulnerabilities": 0, "fim_24h": 0}

                    async def _count(sql, default=0):
                        try:
                            await cur.execute(sql)
                            row = await cur.fetchone()
                            return int(list(row.values())[0]) if row else default
                        except Exception:
                            return default   # table not provisioned yet

                    stats["alerts"] = await _count("SELECT COUNT(*) AS n FROM events_alert")
                    stats["critical"] = await _count(
                        "SELECT COUNT(*) AS n FROM events_alert WHERE severity = 'CRITICAL'"
                    )
                    stats["insights_1h"] = await _count(
                        "SELECT COUNT(*) AS n FROM ai_analysis_results "
                        "WHERE created_at > (NOW() - INTERVAL 1 HOUR)"
                    )
                    stats["vulnerabilities"] = await _count(
                        "SELECT COUNT(*) AS n FROM vulnerabilities_report"
                    )
                    stats["fim_24h"] = await _count(
                        "SELECT COUNT(*) AS n FROM fim_data "
                        "WHERE status IS NOT NULL AND status <> 'baseline' "
                        "AND last_seen > (NOW() - INTERVAL 1 DAY)"
                    )
                    return stats
                finally:
                    await cur.close()

        per_agent = await gather_over_agents(_for_agent, agents)
        scanned = {s["agent"] for s in per_agent}

        totals = {
            key: sum(s[key] for s in per_agent)
            for key in ("alerts", "critical", "insights_1h", "vulnerabilities", "fim_24h")
        }
        # Agents that have actually reported an alert. An enrolled agent that
        # is online but has never reported is the blind spot worth surfacing,
        # which is what makes this different from the online count.
        reporting = sum(1 for s in per_agent if s["alerts"] > 0)

        return sanic_json({
            "status": "success",
            "totals": totals,
            "agents": sorted(per_agent, key=lambda s: (s["critical"], s["alerts"]), reverse=True),
            "coverage": {
                "agents_total": len(agents),
                "agents_scanned": len(per_agent),
                "agents_reporting": reporting,
                "agents_unreachable": sorted(set(agents) - scanned),
                "complete": bool(agents) and len(per_agent) == len(agents),
            },
            "timestamp": datetime.now().isoformat(),
        })
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)


@require_permission("read_telemetry")
@app.route("/api/exposure/report", name="exposure_report")
# Kept so anything already pointing at the old path keeps working. The name is
# wrong for what it returns, which is why it is not the primary route.
@app.route("/api/compliance/report", name="compliance_report_legacy")
async def get_exposure_report(request):
    """Fleet exposure: unpatched packages and recent file-integrity changes.

    This was `/api/compliance/report` and returned
    `100 - (vulns * 2) - (fim * 5)` as a "compliance score". That is not
    compliance — it maps to no framework, so it cannot be shown to an auditor;
    it does not scale with fleet size, so ten vulnerabilities on one agent and
    ten across fifty score identically; and at fifty vulnerabilities it pins to
    zero, which is an ordinary state for a real fleet. A number nobody can
    define is a number nobody can act on.

    It also queried `sentora_hub` for two tables that only exist per-agent, so
    it had never returned anything but a 500. Creating them empty there would
    have made it answer EXCELLENT/100 forever.

    What is reported now is only what is actually measured: counts, per agent,
    with explicit coverage. No score.

    Severity breakdown is deliberately absent — `vulnerabilities_report` has no
    severity column and its fields are encrypted at rest, so any grading would
    have to be invented here.
    """
    try:
        agents = await _list_agent_names_sync()

        async def _for_agent(agent):
            async with agent_conn(agent) as cnx:
                cur = await cnx.cursor()
                try:
                    await cur.execute("SELECT COUNT(*) FROM vulnerabilities_report")
                    row = await cur.fetchone()
                    vulns = int(row[0]) if row else 0

                    # Grouped rather than summed: "12 files changed" and
                    # "12 files deleted" are very different mornings.
                    await cur.execute(
                        "SELECT status, COUNT(*) FROM fim_data "
                        "WHERE status IS NOT NULL AND status <> 'baseline' "
                        "AND last_seen > NOW() - INTERVAL 1 DAY "
                        "GROUP BY status"
                    )
                    fim_rows = await cur.fetchall() or []
                finally:
                    await cur.close()

            fim = {str(s).lower(): int(c) for s, c in fim_rows}
            return {
                "agent": agent,
                "vulnerabilities": vulns,
                "fim_changed": fim.get("changed", 0),
                "fim_new": fim.get("new", 0),
                "fim_deleted": fim.get("deleted", 0),
            }

        # Concurrent: these are independent databases, and walking them in
        # sequence made latency scale with fleet size for no reason.
        per_agent = await gather_over_agents(_for_agent, agents)
        # A newly-enrolled agent may not have provisioned these tables yet.
        # Skipping is correct; hiding it is not — a total over half the fleet
        # is not a fleet total.
        unreachable = sorted(set(agents) - {e["agent"] for e in per_agent})

        totals = {
            key: sum(e[key] for e in per_agent)
            for key in ("vulnerabilities", "fim_changed", "fim_new", "fim_deleted")
        }

        # Worst first: the operator wants to know where to start.
        per_agent.sort(key=lambda a: (a["vulnerabilities"], a["fim_changed"]), reverse=True)

        return sanic_json({
            "status": "success",
            "totals": totals,
            "agents": per_agent,
            "coverage": {
                "agents_total": len(agents),
                "agents_scanned": len(per_agent),
                "agents_unreachable": unreachable,
                # False when the numbers do not cover the whole fleet, so the
                # UI can mark them partial rather than authoritative.
                "complete": bool(agents) and not unreachable,
            },
            "window": "fim counts cover the last 24 hours",
            "timestamp": datetime.now().isoformat(),
        })
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

@require_permission("read_telemetry")
@app.route("/api/assets")
async def get_assets(request):
    """
    Returns consolidated hardware inventory.
    """
    try:
        async with aiomysql.create_pool(host=DB_HOST, port=DB_PORT, user=DB_USER, password=DB_PASSWORD, db="sentora_hub", autocommit=True) as pool:
            async with pool.acquire() as conn:
                async with conn.cursor(aiomysql.DictCursor) as cur:
                    await cur.execute("SELECT * FROM hardware_inventory ORDER BY timestamp DESC")
                    rows = await cur.fetchall()
                    return sanic_json(rows)
    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)



@require_permission("manage_users")
@app.route("/login-logs", methods=["GET"])
async def stream_login_logs(request):
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        await cursor.execute("SELECT * FROM login_logs ORDER BY id DESC")
        columns = [col[0] for col in cursor.description]
        rows = await cursor.fetchall()
        await cursor.close(); await cnx.close()

        json_lines = [
            pyjson.dumps(dict(zip(columns, row)), cls=CustomEncoder, ensure_ascii=False)
            for row in rows
        ]

        return HTTPResponse(
            body='[' + ','.join(json_lines) + ']',
            content_type="application/json; charset=utf-8"
        )
    except Exception as e:
        return sanic_json({"status": "error", "message": f"DB error: {e}"}, status=500)

@app.route("/test-audit", methods=["GET"])
@require_permission("manage_system")
async def test_audit_log(request):
    await audit_log(request, "TEST_ACTION", "TEST_RESOURCE", "Testing audit log insertion")
    return sanic_json({"status": "success", "message": "Test audit log triggered"})

@require_permission("manage_users")
@app.route("/audit-logs", methods=["GET"])
async def stream_audit_logs(request):
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()
        await cursor.execute("SELECT * FROM audit_logs ORDER BY id DESC")
        columns = [col[0] for col in cursor.description]
        rows = await cursor.fetchall()
        await cursor.close(); await cnx.close()

        json_lines = [
            pyjson.dumps(dict(zip(columns, row)), cls=CustomEncoder, ensure_ascii=False)
            for row in rows
        ]

        return HTTPResponse(
            body='[' + ','.join(json_lines) + ']',
            content_type="application/json; charset=utf-8"
        )
    except Exception as e:
        return sanic_json({"status": "error", "message": f"DB error: {e}"}, status=500)


@require_permission("manage_db")
@require_permission("manage_db")
@app.route("/databases", methods=["GET"])
async def list_databases(request):

    try:
        def _q():
            with sync_mysql_conn() as conn:
                cursor = conn.cursor()
                try:
                    cursor.execute("SHOW DATABASES")
                    excluded = {"information_schema", "mysql", "performance_schema", "sys"}
                    return [row[0] for row in cursor.fetchall() if row[0] not in excluded]
                finally:
                    cursor.close()
        all_dbs = await asyncio.to_thread(_q)
        return sanic_json({"status": "success", "databases": all_dbs})
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)

# MySQL: 1049 unknown database, 1146 table doesn't exist. Both mean the
# client named something that isn't there — a 404, not a server fault. They
# were surfacing as 500s, which makes a mistyped name in the DB browser look
# like the platform is broken and buries real faults in the same bucket.
_MISSING_OBJECT_CODES = (1049, 1146)


def _db_object_error(e: Exception):
    code = getattr(e, "errno", None)
    if code is None:
        m = re.search(r"\b(1049|1146)\b", str(e))
        code = int(m.group(1)) if m else None
    if code in _MISSING_OBJECT_CODES:
        return sanic_json({"status": "error", "message": str(e)}, status=404)
    return sanic_json({"status": "error", "message": str(e)}, status=500)


@require_permission("manage_db")
@app.route("/databases/<db_name>/tables", methods=["GET"])
async def list_tables_in_database(request, db_name):
    try:
        def _q():
            with sync_mysql_conn(db_name) as conn:
                cursor = conn.cursor()
                try:
                    cursor.execute("SHOW TABLES")
                    return [row[0] for row in cursor.fetchall()]
                finally:
                    cursor.close()
        tables = await asyncio.to_thread(_q)
        return sanic_json({"status": "success", "tables": tables})
    except Exception as e:
        return _db_object_error(e)

@require_permission("manage_db")
@app.route("/databases/<db_name>/tables/<table_name>/columns", methods=["GET"])
async def list_table_columns(request, db_name, table_name):
    try:
        def _q():
            with sync_mysql_conn(db_name) as conn:
                cursor = conn.cursor()
                try:
                    cursor.execute(f"DESCRIBE {_quote_identifier(table_name)}")
                    return [
                        {"name": row[0], "type": row[1], "null": row[2], "key": row[3], "default": row[4], "extra": row[5]}
                        for row in cursor.fetchall()
                    ]
                finally:
                    cursor.close()
        columns = await asyncio.to_thread(_q)
        return sanic_json({"status": "success", "columns": columns})
    except Exception as e:
        return _db_object_error(e)

@require_permission("manage_db")
@app.route("/databases/<db_name>/tables/<table_name>/data", methods=["GET"])
async def get_table_data(request, db_name, table_name):
    try:
        limit = int(request.args.get("limit", 100))
        def _q():
            with sync_mysql_conn(db_name) as conn:
                cursor = conn.cursor(dictionary=True)
                try:
                    cursor.execute(f"SELECT * FROM {_quote_identifier(table_name)} LIMIT %s", (limit,))
                    return cursor.fetchall()
                finally:
                    cursor.close()
        rows = await asyncio.to_thread(_q)
        
        json_rows = [
            pyjson.dumps(row, cls=CustomEncoder, ensure_ascii=False)
            for row in rows
        ]
        
        return HTTPResponse(
            body='{"status": "success", "data": [' + ','.join(json_rows) + ']}',
            content_type="application/json; charset=utf-8"
        )
    except Exception as e:
        return _db_object_error(e)

@require_permission("manage_users")
@app.route("/roles/<role_id>", methods=["DELETE"])
async def delete_role(request, role_id):
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        await cursor.execute("SELECT role_name FROM roles WHERE id = %s", (role_id,))
        result = await cursor.fetchone()
        if not result:
            await cursor.close(); await cnx.close()
            return sanic_json({"status": "error", "message": "Role not found"}, status=404)

        role_name = result[0]
        if role_name in ["admin"]:
            return sanic_json({"status": "error", "message": f"System-defined role '{role_name}' cannot be deleted."}, status=403)

        await cursor.execute("DELETE FROM role_permissions WHERE role_id = %s", (role_id,))
        await cursor.execute("DELETE FROM roles WHERE id = %s", (role_id,))
        await cnx.commit()

        await cursor.close(); await cnx.close()
        return sanic_json({"status": "success", "message": f"Role '{role_name}' deleted successfully."})
    
    except Exception as e:
        return sanic_json({"status": "error", "message": f"Database error: {e}"}, status=500)

@require_permission("manage_users")
@app.route("/permissions", methods=["GET"])
async def list_permissions(request):
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        await cursor.execute("SELECT id, name, description FROM permissions")
        rows = await cursor.fetchall()
        await cursor.close(); await cnx.close()

        permissions = [
            {
                "id": row[0],
                "name": row[1],
                "description": row[2] or ""
            }
            for row in rows
        ]
        return sanic_json({"status": "success", "permissions": permissions})

    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)




@require_permission("manage_users")
@app.route("/roles", methods=["GET"])
async def list_roles(request):
    try:
        cnx = await connect_userdb()
        cursor = await cnx.cursor()

        await cursor.execute("SELECT id, role_name, created_by, created_at FROM roles")
        role_rows = await cursor.fetchall()

        roles = []
        for role in role_rows:
            role_id, role_name, created_by, created_at = role

            await cursor.execute("""
                SELECT p.name, p.description
                FROM role_permissions rp
                JOIN permissions p ON rp.permission_id = p.id
                WHERE rp.role_id = %s
            """, (role_id,))
            permission_rows = await cursor.fetchall()

            permissions = [
                {"name": perm_name, "description": perm_desc or ""}
                for perm_name, perm_desc in permission_rows
            ]

            roles.append({
                "id": role_id,
                "role_name": role_name,
                "created_by": created_by,
                "created_at": created_at.strftime("%Y-%m-%d %H:%M:%S") if created_at else None,
                "permissions": permissions
            })

        await cursor.close(); await cnx.close()

        return sanic_json({
            "status": "success",
            "roles": roles
        })

    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)


# `_get_agent_keys` was here: the candidate `X-Agent-Key` values to present
# when dialling an agent. The server does not dial one any more, so it has
# no key to offer anybody - the agent presents *its* key when it opens the
# channel, and `_validate_agent_auth_sync` checks that instead.
#
# The lesson it carried is still live in `_linked_agent`. This looked up
# `agent_identities.agent_name`, which holds what the agent enrolled as
# (`DESKTOP-EVS8H9J`), with the name the console arrives with - which
# `server._sanitize_db_name` has already flattened to `DESKTOP_EVS8H9J`.
# Matching exactly found nothing and fell through to the fleet secret,
# which the agent rightly refused, so every call to any host with a hyphen
# in its name returned 401 and the agent looked broken while being correct
# at every step. `_agent_name_forms` is the fix, and it has now been needed
# in four places.


def _relayed_status(upstream_status: int) -> int:
    """Translate a status from an agent into one this API can honestly return.

    401 and 403 are answers about *this API's* caller. An agent rejecting the
    key the server presented is a different sentence entirely - the operator's
    session is fine, and the two systems that failed to authenticate are the
    server and the agent.

    Relaying it unchanged logged the operator out. The browser's interceptor
    treats 401 as "your session ended", clears local storage and redirects to
    the login page, so opening the Config tab of an agent whose key had gone
    stale threw the operator out of the console - with no message, and nothing
    wrong with their session.

    502 is what this is: a bad answer from upstream.
    """
    if upstream_status in (401, 403):
        return 502
    return upstream_status


def _linked_agent(agent: str):
    """The open channel for this agent, under either spelling of its name.

    The channel registers under `agent_identities.agent_name`, which is what
    the agent enrolled as - `DESKTOP-EVS8H9J`. Callers arrive with the name
    the console uses, which comes from the database list and has had its
    hyphens flattened - `DESKTOP_EVS8H9J`.

    Matching exactly would have returned None for every hyphenated host, so
    every request would have fallen back to HTTP and nothing would have said
    why: the feature would have looked implemented and never once fired. Same
    mismatch as `_get_agent_keys`, same fix.
    """
    hyphen, underscore, _ = _agent_name_forms(agent)
    for name in dict.fromkeys([agent, hyphen, underscore]):
        if not name:
            continue
        link = AGENT_LINKS.get(name)
        if link is not None:
            return link
    return None


async def _agent_proxy(agent: str, method: str, path: str, json_body=None,
                       timeout: int = 5):
    """Call one path on an agent, over the channel it holds open.

    Returns `(body, status)` ready to hand to `sanic_json`. Never raises for a
    transport failure: an agent that cannot be reached is an ordinary state of
    the world, and the operator needs the reason rather than a stack trace
    turned into a 500 with no body worth reading.

    This is the chokepoint every server-to-agent feature goes through -
    config, SOAR dispatch, restart, uninstall - so none of the callers has to
    know how the request travelled.

    There used to be an HTTP fallback here that dialled the endpoint on port
    9099. It is gone, and nothing it could rescue is lost with it: the channel
    is an outbound connection to the same server the agent already posts
    telemetry to, so any agent reporting at all can open one. The fallback
    needed an *inbound* path, which is strictly harder - behind NAT or a host
    firewall it never worked, and making it work meant an authenticated
    management API with /self_destruct on it, exposed on every endpoint in the
    fleet.

    What that leaves is a binary predating the channel, which genuinely cannot
    be reached now. That is a real state and it gets a real message, rather
    than a timeout against an address nobody is listening on.
    """
    link = _linked_agent(agent)
    if link is None:
        return ({"status": "error",
                 "message": _no_channel_message(agent),
                 "agent": agent},
                502)
    try:
        return await link.request(method, path, json_body, timeout=timeout)
    except agent_link.LinkError as e:
        print(f"[agent-proxy] {agent}: {method} {path} failed ({e})", flush=True)
        return ({"status": "error",
                 "message": f"The channel to {agent} did not carry this "
                            f"request: {e}",
                 "agent": agent},
                502)


def _no_channel_message(agent: str) -> str:
    """Why a connected-looking agent cannot be commanded.

    Telemetry and control now travel in the same direction, so an agent that
    is sending data and has no channel is almost always running a build from
    before the channel existed. Saying so beats "unreachable", which was true
    of every cause and useful for none.
    """
    return (f"{agent} has no open channel to this server, so there is no way "
            f"to send it a command. Telemetry arrives over a separate "
            f"connection, so an agent can look online here and still be "
            f"uncontrollable.\n\n"
            f"Three things cause this. The agent binary predates the channel - "
            f"rebuild and reinstall it. Or it cannot reach this server's "
            f"/agent-link over ws/wss. Or it holds a key this server no longer "
            f"accepts, in which case the server log has a line reading "
            f"'[agent-link] refused a connection'; restart the agent so it "
            f"re-fetches its key from /api/agents/bootstrap, and if it was "
            f"re-enrolled under a different name, the identity this console "
            f"holds belongs to the old one.")


async def _agent_lifecycle_command(agent: str, path: str, action: str,
                                   comment: str, *, queue_action: str | None = None,
                                   timeout: int = 10) -> tuple[dict, int]:
    """Restart, reload or uninstall an agent, and say which actually happened.

    These three are not SOAR actions and were being sent as if they were. The
    agent's `/soar/execute` accepts only the members of its `ActionType` enum,
    and `self_destruct` is not one - so the direct push came back 501 "action
    not implemented" every time, for a command the agent implements at its own
    `/self_destruct` endpoint. `restart_service` happened to be in the enum,
    which is the whole reason restart appeared to work and uninstall did not.

    Two channels, both kept:

      direct   the dedicated endpoint. Immediate, and it can report a result.
      queued   a row in `automations` the agent picks up on its next poll.
               This is what survives an agent that is asleep, mid-restart or
               briefly unreachable.

    The queue goes first so the command is durable before we try to hurry it,
    and the answer distinguishes the two - "the agent did it" and "the agent
    will do it when it next checks in" are different things to tell someone
    who has just clicked Uninstall.
    """
    queued = False
    try:
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute("""
            INSERT INTO automations
            (device, event_id, action, target, comment, status, `timestamp`, created_at, updated_at)
            VALUES (%s, 0, %s, %s, %s, 'pending', NOW(), NOW(), NOW())
        """, (agent, queue_action or action, "sentora-agent", comment))
        await cnx.commit()
        await cur.close(); await cnx.close()
        queued = True
    except Exception as e:
        print(f"[!] could not queue '{action}' for {agent}: {e}", flush=True)

    body, status = await _agent_proxy(agent, "POST", path, {"comment": comment},
                                      timeout=timeout)

    if status < 400:
        detail = body.get("message") if isinstance(body, dict) else None
        return ({"status": "success", "delivered": "direct", "queued": queued,
                 "message": detail or f"{action}: the agent acted on it."}, 200)

    reason = body.get("message") if isinstance(body, dict) else str(body)
    if queued:
        # Not a failure yet. The row is in the queue and the agent polls it,
        # so saying "failed" here would be wrong in the ordinary case of an
        # agent that is simply not answering this second.
        return ({"status": "pending", "delivered": "queued", "queued": True,
                 "message": f"Could not reach the agent right now, so "
                            f"'{action}' is queued and will run when it next "
                            f"checks in (within a minute). Reason: {reason}",
                 "outcomes": body.get("outcomes") if isinstance(body, dict) else None},
                202)

    return ({"status": "error", "delivered": "none", "queued": False,
             "message": f"Could not reach the agent and could not queue "
                        f"'{action}': {reason}",
             "outcomes": body.get("outcomes") if isinstance(body, dict) else None},
            502)


def _shape_soar_target(action: str, target):
    """Normalize a target value into the shape each SOAR action expects.

    run_cmd on the agent requires a non-empty *list* (argv form); everything
    else takes a single string. Accept strings, lists, JSON-encoded lists,
    and shape consistently.

    CRITICAL: must be idempotent. If a previous trigger already JSON-encoded
    the list to `'["dir"]'` and stored it in the automations table, the next
    poll cycle that re-shapes the value MUST NOT wrap it again (otherwise
    each retry escapes the quotes once more and the target eventually
    explodes into a multi-kilobyte chain of backslashes).
    """
    act = (action or "").strip().lower()
    if act == "run_cmd":
        # already a list — just clean it
        if isinstance(target, list):
            return [str(x) for x in target if x is not None and str(x) != ""]
        s = "" if target is None else str(target).strip()
        if not s:
            return []
        # JSON-encoded list ('["dir"]') from a previous shaping round — unwrap
        # and keep unwrapping while we still find a nested JSON list, so we
        # never double-escape.
        for _ in range(6):
            if not (s.startswith('[') and s.endswith(']')):
                break
            try:
                parsed = pyjson.loads(s)
            except Exception:
                break
            if not isinstance(parsed, list):
                break
            cleaned = [str(x) for x in parsed if x is not None and str(x) != ""]
            if len(cleaned) == 1 and cleaned[0].startswith('[') and cleaned[0].endswith(']'):
                s = cleaned[0]
                continue
            return cleaned
        # plain shell-style string — split with shlex (POSIX off for Windows paths)
        import shlex
        try:
            return shlex.split(s, posix=False)
        except Exception:
            return [s]
    if isinstance(target, list):
        return " ".join(str(x) for x in target).strip()
    return str(target).strip() if target is not None else ""


async def call_agent_soar(
    agent: str,
    action: str,
    target,
    *,
    comment: str = "",
    event_id: int | None = None,
    ttl: int | None = None,
    background_queue: bool = True
) -> dict:
    """Call the agent's /soar/execute.

    With `background_queue` (the default) the action is also written to the
    database, so an agent that missed the direct push still picks it up when
    it next polls - which is what makes this survive a restart or a dropped
    connection.
    """
    target = _shape_soar_target(action, target)

    if background_queue:
        try:
            eid = int(event_id) if event_id is not None else 0
            db_target = pyjson.dumps(target) if isinstance(target, list) else target
            cnx = await connect_db_for_agent(agent)
            cur = await cnx.cursor()
            await cur.execute("""
                INSERT INTO automations
                (device, event_id, action, target, comment, status, `timestamp`, created_at, updated_at)
                VALUES (%s, %s, %s, %s, %s, 'pending', NOW(), NOW(), NOW())
            """, (agent, eid, action, db_target, comment))
            await cnx.commit()
            await cur.close(); await cnx.close()
        except Exception as e:
            print(f"[!] Background skip: Failed to queue '{action}' for {agent}: {e}")

    payload: dict = {
        "action": action,
        "target": target,
    }
    if comment: payload["comment"] = comment
    if event_id is not None: payload["event_id"] = int(event_id)
    if ttl is not None: payload["ttl"] = int(ttl)

    try:
        # Through `_agent_proxy`, so SOAR dispatch gets what the config tab
        # already had: every known address tried rather than the first, keys
        # resolved under both spellings of the agent's name, and a failure
        # that names which address did what. This used a single address and a
        # single guess at the name, which is why dispatch failed on hosts the
        # console could otherwise talk to.
        data, status = await _agent_proxy(agent, "POST", "/soar/execute",
                                          payload, timeout=5)
        data = data if isinstance(data, dict) else {}

        http_ok = 200 <= status < 300
        agent_ok = bool(data.get("ok", True))

        if http_ok and agent_ok:
            return {
                "ok": True,
                "queued": background_queue,
                "message": data.get("message", "Action pushed to agent"),
                "soar_action_id": data.get("soar_action_id"),
                **data
            }
        msg = data.get("error") or data.get("message") or f"HTTP {status}"
        return {
            "ok": False,
            "queued": background_queue,
            "message": (f"Direct push failed ({msg})."
                        + (" The agent will pick it up on its next poll."
                           if background_queue else "")),
            "error": msg,
            "outcomes": data.get("outcomes"),
        }

    except Exception as e:
        # Three unreachable `except requests.*` clauses used to follow a
        # catch-all here, referring to a `url` variable that no longer
        # existed. They were dead the moment the catch-all was added.
        #
        # The catch-all also returned `ok: True` on an exception, on the
        # grounds that the command was queued. Whether it was queued is
        # already its own field; conflating that with success is what let a
        # push nobody received be logged as one that worked.
        print(f"[!] SOAR call to {agent} failed: {e}", flush=True)
        return {
            "ok": False,
            "queued": background_queue,
            "message": (f"Could not push to the agent ({e})."
                        + (" The command is queued and will run on its next"
                           " poll." if background_queue else "")),
            "error": str(e),
        }

async def ensure_playbooks_table(agent: str):
    """Create <agent>_db.playbooks if it does not exist, and migrate an old one.

    Called from every playbook route - list, fetch, save, delete, run - so
    this runs on each request, which is what made the two ALTERs that used to
    live here expensive. Both columns are in the CREATE, so on any database
    that already had them each statement failed with a duplicate-column error
    behind `except Exception: pass`: two swallowed errors and, more to the
    point, two metadata locks on `playbooks` per request.

    MySQL takes that lock whether or not the statement changes anything, and
    a waiting DDL blocks every reader queued behind it - the failure that made
    `/automations/pending` stop answering agents. information_schema takes no
    lock, so a table that is already correct is left alone.
    """
    cnx = await connect_db_for_agent(agent)
    cur = await cnx.cursor()
    await cur.execute("""
        CREATE TABLE IF NOT EXISTS playbooks (
            id INT AUTO_INCREMENT PRIMARY KEY,
            agent_name VARCHAR(255) NOT NULL,
            name VARCHAR(255) NOT NULL UNIQUE,
            description TEXT,
            nodes LONGTEXT NOT NULL,
            connections LONGTEXT NOT NULL,
            created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
            updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
    """)

    await cur.execute(
        "SELECT column_name FROM information_schema.columns "
        "WHERE table_schema = DATABASE() AND table_name = 'playbooks'")
    present = {str(r[0]).lower() for r in await cur.fetchall()}
    pending = [ddl for name, ddl in (
        ("description", "ADD COLUMN description TEXT AFTER name"),
        ("updated_at", "ADD COLUMN updated_at TIMESTAMP NOT NULL "
                       "DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"),
    ) if name not in present]
    if present and pending:
        try:
            # Never wait on the lock: the next request tries again, and
            # blocking here stalls the playbook views themselves.
            await cur.execute("SET SESSION lock_wait_timeout = 3")
            await cur.execute(f"ALTER TABLE playbooks {', '.join(pending)}")
            print(f"[+] {agent}: playbooks gained {len(pending)} column(s)",
                  flush=True)
        except Exception as e:
            # Said out loud. A column that never arrives becomes a failing
            # SELECT in a route somewhere else, which is how the swallowed
            # version of this hid.
            print(f"[!] {agent}: could not migrate playbooks ({e}); "
                  f"will retry on the next request", flush=True)

    await cnx.commit()
    await cur.close(); await cnx.close()


def _parse_json_field(val):
    """Turn a nodes/connections column back into a dict or list.

    Stored as JSON text, and returned as text by some drivers and already
    parsed by others.
    """
    if isinstance(val, (dict, list)):
        return val
    try:
        return pyjson.loads(val) if isinstance(val, str) else val
    except Exception:
        return val


@require_permission("manage_soar")
@app.route("/<agent>/playbooks", methods=["GET"])
async def list_playbooks(request, agent):
    """
    Frontend: to populate the list. 
    Nodes and connections are returned too, so opening the editor does not
    lose them.
    """
    try:
        await ensure_playbooks_table(agent)
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        
        try:
            await cur.execute(
                "SELECT id, name, description, nodes, connections, created_at, updated_at FROM playbooks ORDER BY updated_at DESC"
            )
        except Exception:
            await cur.execute(
                "SELECT id, name, '' as description, nodes, connections, created_at, created_at as updated_at FROM playbooks ORDER BY created_at DESC"
            )
        cols = [c[0] for c in cur.description]
        rows = await cur.fetchall()
        await cur.close(); await cnx.close()

        data = []
        for r in rows:
            pb = dict(zip(cols, r))
            pb["nodes"] = _parse_json_field(pb.get("nodes"))
            pb["connections"] = _parse_json_field(pb.get("connections"))
            data.append(pb)

        body = pyjson.dumps(data, cls=CustomEncoder, ensure_ascii=False)
        return HTTPResponse(body=body, content_type="application/json; charset=utf-8")
    except Exception as e:
        return sanic_json({"status": "error", "message": f"{e}"}, status=500)


@require_permission("manage_soar")
@app.route("/<agent>/playbooks/<playbook_id:int>", methods=["GET"])
async def get_playbook(request, agent, playbook_id):
    try:
        await ensure_playbooks_table(agent)
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute(
            "SELECT id, name, description, nodes, connections, created_at, updated_at FROM playbooks WHERE id=%s LIMIT 1",
            (int(playbook_id),)
        )
        row = await cur.fetchone()
        await cur.close(); await cnx.close()

        if not row:
            return sanic_json({"status": "error", "message": "Playbook not found"}, status=404)

        cols = ["id", "name", "description", "nodes", "connections", "created_at", "updated_at"]
        pb = dict(zip(cols, row))
        pb["nodes"] = _parse_json_field(pb["nodes"])
        pb["connections"] = _parse_json_field(pb["connections"])

        body = pyjson.dumps(pb, cls=CustomEncoder, ensure_ascii=False)
        return HTTPResponse(body=body, content_type="application/json; charset=utf-8")
    except Exception as e:
        return sanic_json({"status": "error", "message": f"{e}"}, status=500)


@require_permission("manage_soar")
@app.route("/<agent>/playbooks", methods=["POST"])
async def save_playbook(request, agent):
    try:
        await ensure_playbooks_table(agent)
        data = request.json or {}

        name = (data.get("name") or "").strip()
        description = data.get("description") or ""
        if not name:
            return sanic_json({"status": "error", "message": "name is required"}, status=400)

        nodes_json = pyjson.dumps(data.get("nodes") or [], ensure_ascii=False)
        conns_json = pyjson.dumps(data.get("connections") or [], ensure_ascii=False)

        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()

        pid = data.get("id")
        if pid:
            await cur.execute(
                """
                UPDATE playbooks 
                SET name=%s, description=%s, nodes=%s, connections=%s, updated_at=NOW() 
                WHERE id=%s
                """,
                (name, description, nodes_json, conns_json, int(pid))
            )
            await cnx.commit()
            updated = cur.rowcount
            await cur.close(); await cnx.close()
            if updated == 0:
                return sanic_json({"status": "error", "message": "Playbook not found"}, status=404)
            return sanic_json({"status": "success", "id": int(pid), "mode": "updated"}, status=200)

        await cur.execute("SELECT id FROM playbooks WHERE name=%s LIMIT 1", (name,))
        row = await cur.fetchone()
        if row:
            pid = int(row[0])
            await cur.execute(
                """
                UPDATE playbooks 
                SET description=%s, nodes=%s, connections=%s, updated_at=NOW() 
                WHERE id=%s
                """,
                (description, nodes_json, conns_json, pid)
            )
            await cnx.commit()
            await cur.close(); await cnx.close()
            return sanic_json({"status": "success", "id": pid, "mode": "updated"}, status=200)
        else:
            await cur.execute(
                """
                INSERT INTO playbooks (agent_name, name, description, nodes, connections, created_at, updated_at)
                VALUES (%s, %s, %s, %s, %s, NOW(), NOW())
                """,
                (agent, name, description, nodes_json, conns_json)
            )
            pid = cur.lastrowid
            await cnx.commit()
            await cur.close(); await cnx.close()
            return sanic_json({"status": "success", "id": int(pid), "mode": "created"}, status=201)
    except Exception as e:
        return sanic_json({"status": "error", "message": f"{e}"}, status=500)




@require_permission("manage_soar")
@app.route("/<agent>/playbooks/<playbook_id:int>", methods=["PUT"])
async def update_playbook(request, agent, playbook_id):
    try:
        await ensure_playbooks_table(agent)
        payload = request.json or {}
        name = (payload.get("name") or "").strip()
        description = payload.get("description") or ""
        nodes_json = pyjson.dumps(payload.get("nodes") or [], ensure_ascii=False)
        conns_json = pyjson.dumps(payload.get("connections") or [], ensure_ascii=False)

        if not name:
            return sanic_json({"status": "error", "message": "name is required"}, status=400)

        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute(
            """
            UPDATE playbooks 
            SET name=%s, description=%s, nodes=%s, connections=%s, updated_at=NOW() 
            WHERE id=%s
            """,
            (name, description, nodes_json, conns_json, int(playbook_id))
        )
        await cnx.commit()
        await cur.close(); await cnx.close()
        return sanic_json({"status": "success", "message": "Updated"})
    except Exception as e:
        return sanic_json({"status": "error", "message": f"{e}"}, status=500)

        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute(
            "UPDATE playbooks SET name=%s, nodes=%s, connections=%s, updated_at=NOW() WHERE id=%s",
            (name, nodes_json, conns_json, int(playbook_id))
        )
        await cnx.commit()
        changed = cur.rowcount
        await cur.close(); await cnx.close()

        if changed == 0:
            return sanic_json({"status": "error", "message": "Playbook not found"}, status=404)
        return sanic_json({"status": "success", "id": int(playbook_id)}, status=200)

    except Exception as e:
        return sanic_json({"status": "error", "message": f"{e}"}, status=500)


@require_permission("manage_soar")
@app.delete("/<agent>/playbooks/<playbook_id:int>")
async def delete_playbook(request, agent, playbook_id):
    """
    Delete a playbook by its ID.  Returns a JSON payload with status and
    number of deleted rows (0 or 1).
    """
    try:
        await ensure_playbooks_table(agent)
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute(
            "DELETE FROM playbooks WHERE id=%s",
            (int(playbook_id),)
        )
        await cnx.commit()
        deleted = cur.rowcount
        await cur.close(); await cnx.close()
        if not deleted:
            return sanic_json({"status": "error", "message": "playbook not found"}, status=404)
        return sanic_json({"status": "success", "deleted": int(playbook_id)})
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)





# The node palette moved to core/soar_palette.py: it is a static
# catalogue describing the SOAR builder UI, not application logic.
from core import soar_palette  # noqa: E402

soar_palette.set_defaults(soar_enabled=is_soar_enabled)
_build_node_palette = soar_palette._build_node_palette


@app.get("/playbooks/palette")
async def get_node_palette(request):
    """UI paletini doldurur."""
    return sanic_json(_build_node_palette())

@app.get("/playbooks/examples")
async def get_playbook_examples(request):
    """A few ready-made playbooks, so the editor is not empty on first use."""
    examples = [
        {
            "name": "Critical Alert → Block IP → Email",
            "nodes": [
                {"id": "t1", "type": "trigger.events_alert", "config": {"min_severity": "CRITICAL"}},
                {"id": "c1", "type": "condition.severity_at_least", "config": {"threshold": "CRITICAL"}},
                {"id": "a1", "type": "action.soar.block_ip", "config": {"ip": "{{event.ip}}"}},
                {"id": "n1", "type": "notify.email", "config": {"template_name": "Critical Alerts - Agent: {{agent}}", "context_json": "{\"agent\":\"{{agent}}\",\"body\":\"Blocked {{event.ip}}\"}"}}
            ],
            "connections": [
                {"from": "t1", "to": "c1"},
                {"from": "c1", "to": "a1"},
                {"from": "a1", "to": "n1"}
            ]
        }
    ]
    return sanic_json({"examples": examples})

@app.post("/<agent>/playbooks/<playbook_id:int>/webhook")
@require_permission("manage_soar")
async def playbook_webhook(request, agent, playbook_id):
    """
    This endpoint allows external systems to trigger a playbook via webhook.
    For now it acknowledges the call. In a real implementation this could look up
    the playbook, evaluate trigger conditions, and enqueue a run based on the
    provided event payload.
    """
    try:
        event = request.json or {}
        return sanic_json({"ok": True, "message": "Webhook received", "event": event})
    except Exception as e:
        return sanic_json({"ok": False, "error": str(e)}, status=500)


def _validate_graph_payload(payload: dict):
    errs = []
    nodes = payload.get("nodes") or []
    conns = payload.get("connections") or []
    known_types = {n["type"] for n in _build_node_palette()["nodes"]}

    ids = set()
    for i, n in enumerate(nodes):
        nid = n.get("id")
        ntype = n.get("type")
        if not nid or not isinstance(nid, str):
            errs.append(f"nodes[{i}].id missing/invalid")
        elif nid in ids:
            errs.append(f"duplicate node id: {nid}")
        else:
            ids.add(nid)
        if not ntype or ntype not in known_types:
            errs.append(f"nodes[{i}].type unknown: {ntype}")

    for i, c in enumerate(conns):
        f = c.get("from"); t = c.get("to")
        if f not in ids or t not in ids:
            errs.append(f"connections[{i}] invalid endpoints: {f}->{t}")

    return errs



def _render_with_ctx(template_str: str, ctx: dict) -> str:
    try:
        return render_template(template_str, ctx)
    except Exception:
        return template_str

async def _ensure_playbook_runs_table(agent: str):
    cnx = await connect_db_for_agent(agent)
    cur = await cnx.cursor()
    await cur.execute("""
        CREATE TABLE IF NOT EXISTS playbook_runs (
            id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
            agent_name    VARCHAR(255) NOT NULL,
            playbook_name VARCHAR(255) NOT NULL,
            status ENUM('running','success','failed','cancelled') NOT NULL DEFAULT 'running',
            started_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
            finished_at TIMESTAMP NULL,
            timeline   LONGTEXT NULL,
            last_error TEXT NULL,
            KEY idx_runs_agent_started (agent_name, started_at),
            KEY idx_runs_status (status)
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
    """)
    await cnx.commit()
    await cur.close(); await cnx.close()

async def _append_run_log(agent: str, run_id: int, timeline: list, status: str = None):
    cnx = await connect_db_for_agent(agent)
    cur = await cnx.cursor()
    tjson = pyjson.dumps(timeline, ensure_ascii=False, cls=CustomEncoder)
    if status:
        await cur.execute(
            "UPDATE playbook_runs SET timeline=%s, status=%s, finished_at=NOW() WHERE id=%s",
            (tjson, status, run_id)
        )
    else:
        await cur.execute("UPDATE playbook_runs SET timeline=%s WHERE id=%s", (tjson, run_id))
    await cnx.commit()
    await cur.close(); await cnx.close()

def _sev_rank(v: str) -> int:
    order = {"LOW":1,"MEDIUM":2,"HIGH":3,"CRITICAL":4}
    return order.get((v or "").upper(), 0)

def _eval_condition(ntype: str, config: dict, ctx: dict) -> bool:
    if ntype == "condition.severity_at_least":
        thr = (config or {}).get("threshold","HIGH")
        sev = (ctx.get("event") or {}).get("severity","LOW")
        return _sev_rank(sev) >= _sev_rank(thr)
    if ntype == "condition.text_match":
        field = (config or {}).get("field","message")
        needle = (config or {}).get("needle","")
        val = str(((ctx.get("event") or {}).get(field)) or "")
        return (needle or "") in val
    return True


@app.get("/<agent>/playbooks/catalog")
@require_permission("manage_soar")
async def get_playbooks_catalog(request, agent):
    try:
        catalog = {
            "triggers": [
                {"key": "manual", "label": "Manual Trigger", "description": "Start manually"},
                {"key": "schedule", "label": "Scheduled (CRON)", "description": "Time-based trigger"},
                {"key": "webhook", "label": "Webhook", "description": "HTTP callback trigger"},
                {"key": "events_alert", "label": "Events Alert", "description": "Trigger on alert"}
            ],
                "actions": [
                     {"key": "http_request", "label": "HTTP Request", "description": "Make HTTP call"},
                     {"key": "block_ip", "label": "Block IP (SOAR)", "description": "Block IP address"},
                     {"key": "unblock_ip", "label": "Unblock IP (SOAR)", "description": "Unblock IP address"},
                     {"key": "disable_user", "label": "Disable User (SOAR)", "description": "Disable user account"},
                     {"key": "enable_user", "label": "Enable User (SOAR)", "description": "Enable user account"},
                     {"key": "kill_process", "label": "Kill Process (SOAR)", "description": "Kill a running process"},
                     {"key": "restart_service", "label": "Restart Service (SOAR)", "description": "Restart a system service"},
                     {"key": "lock_machine", "label": "Lock Machine (SOAR)", "description": "Lock the machine"},
                     {"key": "quarantine_file", "label": "Quarantine File (SOAR)", "description": "Quarantine a file"},
                     {"key": "tail_log", "label": "Tail Log (SOAR)", "description": "Tail system logs"},
                     {"key": "run_cmd", "label": "Run Command (SOAR)", "description": "Execute a shell command"}
                 ],
            "notifications": [
                {"key": "email", "label": "Email", "description": "Send email"},
                {"key": "slack", "label": "Slack", "description": "Send Slack message"},
                {"key": "teams", "label": "MS Teams", "description": "Send Teams notification"}
            ],
            "scripts": [
                {"key": "python", "label": "Python Script"},
                {"key": "bash", "label": "Bash Script"},
                {"key": "powershell", "label": "PowerShell Script"}
            ],
            "fields": []
        }
        
        return sanic_json(catalog)
    except Exception as e:
        return sanic_json({"error": str(e)}, status=500) 
        
@app.get("/<agent>/notifications/templates")
@require_permission("set_email_config")
async def get_email_templates(request, agent):
    """
    Email template list for notification nodes.
    """
    try:
        cnx = await connect_userdb()
        cur = await cnx.cursor()
        await cur.execute("SELECT id, template_name as name FROM email_templates ORDER BY template_name")
        rows = await cur.fetchall()
        await cur.close()
        await cnx.close()
        
        templates = [{"id": row[0], "name": row[1]} for row in rows]
        return sanic_json(templates)
    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)

@app.post("/<agent>/playbooks/validate")
@require_permission("manage_soar")
async def validate_playbook(request, agent):
    try:
        data = request.json or {}
        nodes = data.get("nodes", [])
        connections = data.get("connections", [])
        
        issues = []
        
        triggers = [n for n in nodes if n.get("type") == "trigger"]
        if not triggers:
            issues.append({
                "type": "error",
                "message": "Workflow must have at least one trigger node"
            })
        
        node_ids = {n["id"] for n in nodes}
        connected = set()
        for c in connections:
            connected.add(c.get("from"))
            connected.add(c.get("to"))
        
        for node in nodes:
            nid = node["id"]
            if nid not in connected and node.get("type") != "trigger":
                issues.append({
                    "type": "warning",
                    "message": f"Node '{node.get('data', {}).get('name', nid)}' is not connected",
                    "nodeId": nid
                })
        
        for node in nodes:
            if node.get("type") == "script":
                cfg = node.get("data", {}).get("config", {})
                if not cfg.get("scriptContent", "").strip():
                    issues.append({
                        "type": "error",
                        "message": f"Script node '{node.get('data', {}).get('name', node['id'])}' has no script content",
                        "nodeId": node["id"]
                    })
        
        for node in nodes:
            if node.get("type") == "action":
                cfg = node.get("data", {}).get("config", {})
                action_type = cfg.get("actionType", "http_request")
                
                if action_type == "http_request" and not cfg.get("url"):
                    issues.append({
                        "type": "error",
                        "message": f"HTTP action '{node.get('data', {}).get('name', node['id'])}' requires URL",
                        "nodeId": node["id"]
                    })
                
                if action_type in ["block_ip", "disable_user"] and not cfg.get("target"):
                    issues.append({
                        "type": "error",
                        "message": f"SOAR action '{node.get('data', {}).get('name', node['id'])}' requires target",
                        "nodeId": node["id"]
                    })
        
        return sanic_json({"issues": issues})
    
    except Exception as e:
        return sanic_json({"error": str(e)}, status=500)


@app.post("/_proxy/http")
@require_permission("manage_system")
async def http_proxy(request):
    """Proxy a UI-originated HTTP request (works around CORS on third-party
    endpoints used by playbook nodes).

    Inert until `PROXY_ALLOWED_HOSTS` names destinations. Every request is
    checked by `security.ssrf.check_target` first — scheme, host allowlist,
    and the resolved address — and both the outbound and returned headers are
    filtered. See that module for the reasoning behind each layer.
    """
    data = request.json or {}
    url = data.get("url")

    method = (data.get("method") or "GET").upper()
    if method not in ssrf.ALLOWED_METHODS:
        return sanic_json({"error": f"Method not allowed: {method}"}, status=400)

    error = ssrf.check_target(url)
    if error:
        # Denials are audit-logged: a burst of them is somebody probing the
        # proxy, and that is worth being able to see after the fact.
        await audit_log(request, "PROXY_DENIED", str(url)[:255], error)
        return sanic_json({"error": error}, status=403)

    headers = ssrf.clean_request_headers(data.get("headers"))
    body = data.get("body")
    timeout = ssrf.clamp_timeout(data.get("timeout", 15))
    max_bytes = ssrf.max_response_bytes()

    await audit_log(request, "PROXY_REQUEST", str(url)[:255], f"{method} timeout={timeout}s")

    def _make_request():
        return requests.request(
            method=method,
            url=url,
            headers=headers,
            data=body if method not in ("GET", "HEAD") else None,
            timeout=timeout,
            # An allowlisted host answering 302 -> 169.254.169.254 would walk
            # straight past every check above, so hops are not followed. The
            # caller gets the redirect response and can re-submit the target,
            # which puts it back through validation.
            allow_redirects=False,
            stream=True,
        )

    try:
        resp = await asyncio.to_thread(_make_request)
    except requests.Timeout:
        return sanic_json({"error": "Request timeout"}, status=504)
    except Exception as e:
        return sanic_json({"error": str(e)}, status=502)

    try:
        # Read one byte past the cap so an oversized body is detectable
        # without buffering the whole thing. decode_content undoes any
        # transport compression, since the framing headers are dropped below.
        content = await asyncio.to_thread(resp.raw.read, max_bytes + 1, decode_content=True)
    except Exception as e:
        return sanic_json({"error": f"Failed reading response: {e}"}, status=502)
    finally:
        await asyncio.to_thread(resp.close)

    if len(content) > max_bytes:
        return sanic_json({"error": f"Response exceeds {max_bytes} bytes"}, status=502)

    return HTTPResponse(
        body=content,
        # A third party's 401 is not this session's 401. Relayed unchanged it
        # would hit the browser's interceptor and log the operator out because
        # somebody else's API wanted credentials - so the status is mapped and
        # the real one is handed back in a header for the playbook to read.
        status=_relayed_status(resp.status_code),
        headers={
            **ssrf.clean_response_headers(resp.headers),
            "X-Upstream-Status": str(resp.status_code),
        },
        content_type=resp.headers.get("Content-Type", "application/octet-stream"),
    )



@require_permission("manage_agent")
@app.post("/<agent>/playbooks/execute")
async def playbook_execute_real(request, agent):
    if not is_soar_enabled():
        return _soar_block_response()

    payload = request.json or {}
    errs = _validate_graph_payload(payload)
    if errs:
        return sanic_json({"ok": False, "errors": errs}, status=400)

    await _ensure_playbook_runs_table(agent)
    cnx = await connect_db_for_agent(agent)
    cur = await cnx.cursor()
    pb_name = (payload.get("name") or "unnamed")[:255]
    await cur.execute(
        "INSERT INTO playbook_runs (agent_name, playbook_name) VALUES (%s, %s)",
        (agent, pb_name),
    )
    run_id = cur.lastrowid
    await cnx.commit()
    await cur.close(); await cnx.close()

    ctx = {
        "agent": agent,
        "event": payload.get("sample_event") or {"ip":"10.10.10.10","username":"alice","severity":"HIGH","message":"sample"},
    }

    timeline = []
    try:
        for n in payload.get("nodes", []):
            ntype = n["type"]
            conf  = n.get("config") or {}
            step  = {"node": n["id"], "type": ntype, "ok": True}

            if ntype.startswith("trigger."):
                step["info"] = "triggered"
                timeline.append(step); continue

            if ntype.startswith("condition."):
                res = _eval_condition(ntype, conf, ctx)
                step["result"] = bool(res)
                timeline.append(step)
                if not res:
                    pass
                continue

            if ntype == "util.delay":
                ms = int(conf.get("ms", 1000))
                ms = max(0, min(ms, 5000))
                await asyncio.sleep(ms/1000.0)
                step["result"] = f"waited {ms}ms"
                timeline.append(step); continue

            if ntype == "notify.email":
                tname = conf.get("template_name") or f"Critical Alerts - Agent: {agent}"
                ctx_str = conf.get("context_json") or "{}"
                try:
                    raw_ctx = pyjson.loads(_render_with_ctx(ctx_str, ctx)) if ctx_str.strip().startswith("{") else {"body": _render_with_ctx(ctx_str, ctx)}
                except Exception:
                    raw_ctx = {"body": _render_with_ctx(ctx_str, ctx)}
                ok = send_email(_render_with_ctx(tname, ctx), raw_ctx,
                                fallback=DEFAULT_ALERT_TEMPLATE)
                step["result"] = "sent" if ok else "failed"
                step["ok"] = bool(ok)
                timeline.append(step); continue

            if ntype == "action.soar.block_ip":
                ip = _render_with_ctx(str(conf.get("ip") or "{{event.ip}}"), ctx)
                r = await call_agent_soar(agent, action="block_ip", target=ip, comment=f"playbook#{run_id}:{pb_name}")
                step["agent_response"] = r
                step["ok"] = bool(r.get("ok"))
                timeline.append(step); continue

            if ntype == "action.soar.disable_user":
                username = _render_with_ctx(str(conf.get("username") or "{{event.username}}"), ctx)
                r = await call_agent_soar(agent, action="disable_user", target=username, comment=f"playbook#{run_id}:{pb_name}")
                step["agent_response"] = r
                step["ok"] = bool(r.get("ok"))
                timeline.append(step); continue

            if ntype.startswith("action.soar."):
                action_name = ntype.split(".")[-1]
                tgt_raw = (
                    conf.get("target") or
                    conf.get("username") or
                    conf.get("ip") or
                    conf.get("pid") or
                    conf.get("service") or
                    conf.get("machine") or
                    conf.get("filepath") or
                    conf.get("logfile") or
                    conf.get("command") or
                    ""
                )
                target_val = _render_with_ctx(str(tgt_raw), ctx)
                r = await call_agent_soar(agent, action=action_name, target=target_val, comment=f"playbook#{run_id}:{pb_name}")
                step["agent_response"] = r
                step["ok"] = bool(r.get("ok"))
                timeline.append(step); continue

            step["ok"] = False
            step["error"] = f"unknown node type: {ntype}"
            timeline.append(step)

        await _append_run_log(
            agent, run_id, timeline,
            status="success" if all(s.get("ok") for s in timeline) else "failed"
        )
        return sanic_json({"ok": True, "run_id": run_id, "timeline": timeline})

    except Exception as e:
        timeline.append({"node": None, "type": "runtime", "ok": False, "error": str(e)})
        await _append_run_log(agent, run_id, timeline, status="failed")
        return sanic_json({"ok": False, "run_id": run_id, "error": str(e), "timeline": timeline}, status=500)
    
@require_permission("manage_agent")
@app.get("/<agent>/playbooks/runs")
async def list_playbook_runs(request, agent):
    """
    Query params:
      - status: running|completed|failed
      - order: ASC|DESC (default DESC)
      - limit: int (default 50, max 1000)
    """
    q = request.args
    status = (q.get("status") or "").strip().lower()
    order = (q.get("order") or "DESC").strip().upper()
    if order not in ("ASC", "DESC"):
        order = "DESC"

    try:
        limit = int(q.get("limit", 50))
        limit = max(1, min(limit, 1000))
    except Exception:
        limit = 50

    where = []
    params = []

    if status:
        where.append("status = %s")
        params.append(status)

    await _ensure_playbook_runs_table(agent)
    cnx = await connect_db_for_agent(agent)
    cur = await cnx.cursor()

    sql = "SELECT id, playbook_name, status, started_at, finished_at, last_error FROM playbook_runs"
    if where:
        sql += " WHERE " + " AND ".join(where)
    sql += f" ORDER BY id {order} LIMIT %s"
    params.append(limit)

    try:
        await cur.execute(sql, tuple(params))
    except Exception:
        await cur.close(); await cnx.close()
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        sql_fallback = "SELECT id, playbook_name, status, started_at, finished_at, NULL as last_error FROM playbook_runs"
        if where:
            sql_fallback += " WHERE " + " AND ".join(where)
        sql_fallback += f" ORDER BY id {order} LIMIT %s"
        await cur.execute(sql_fallback, tuple(params))
    cols = [c[0] for c in cur.description]
    rows = await cur.fetchall()
    await cur.close(); await cnx.close()

    data = [dict(zip(cols, r)) for r in rows]
    body = pyjson.dumps(data, ensure_ascii=False, cls=CustomEncoder)
    return HTTPResponse(body=body, content_type="application/json; charset=utf-8")


@require_permission("manage_agent")
@app.delete("/<agent>/playbooks/runs/<run_id:int>")
async def delete_playbook_run(request, agent, run_id):
    await _ensure_playbook_runs_table(agent)
    cnx = await connect_db_for_agent(agent)
    cur = await cnx.cursor()
    await cur.execute("DELETE FROM playbook_runs WHERE id=%s", (int(run_id),))
    await cnx.commit()
    deleted = cur.rowcount
    await cur.close(); await cnx.close()

    if not deleted:
        return sanic_json({"status": "error", "message": "run not found"}, status=404)
    return sanic_json({"status": "success", "deleted": int(run_id)})

@require_permission("manage_agent")
@app.get("/<agent>/playbooks/runs/<run_id:int>/download")
async def download_playbook_run(request, agent, run_id):
    """
    Download one run's detail as a JSON file.
    """
    await _ensure_playbook_runs_table(agent)
    cnx = await connect_db_for_agent(agent)
    cur = await cnx.cursor()
    await cur.execute(
        "SELECT id, playbook_name, status, started_at, finished_at, timeline FROM playbook_runs WHERE id=%s",
        (int(run_id),)
    )
    row = await cur.fetchone()
    await cur.close(); await cnx.close()

    if not row:
        return sanic_json({"status": "error", "message": "run not found"}, status=404)

    cols = ["id", "playbook_name", "status", "started_at", "finished_at", "timeline"]
    d = dict(zip(cols, row))

    tl = d.get("timeline")
    if isinstance(tl, (bytes, bytearray)):
        tl = tl.decode("utf-8", "ignore")
    if isinstance(tl, str):
        try:
            d["timeline"] = pyjson.loads(tl)
        except Exception:
            pass

    payload = pyjson.dumps(d, ensure_ascii=False, cls=CustomEncoder)
    filename = f"playbook_run_{run_id}.json"
    headers = {
        "Content-Type": "application/json; charset=utf-8",
        "Content-Disposition": f'attachment; filename="{filename}"'
    }
    return HTTPResponse(body=payload, headers=headers)


@app.post("/<agent>/playbooks/<playbook_id:int>/run")
@require_permission("manage_soar")
async def run_playbook(request, agent, playbook_id):
    if not is_soar_enabled():
        return _soar_block_response()

    import traceback as _tb
    import json as pyjson

    run_id = None
    timeline: list = []

    try:
        await ensure_playbooks_table(agent)
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute("SELECT nodes, connections, name FROM playbooks WHERE id=%s", (playbook_id,))
        row = await cur.fetchone()
        await cur.close(); await cnx.close()
        if not row:
            return sanic_json({"ok": False, "error": "playbook not found"}, status=404)

        raw_nodes = _parse_json_field(row[0])
        raw_conns = _parse_json_field(row[1])
        nodes = raw_nodes if isinstance(raw_nodes, list) else []
        conns = raw_conns if isinstance(raw_conns, list) else []
        pb_name = (row[2] or "unnamed")[:255]

        nexts: dict = {}
        for c in conns:
            if not isinstance(c, dict):
                continue
            f, t = c.get("from"), c.get("to")
            if f and t:
                nexts.setdefault(f, []).append(t)

        await _ensure_playbook_runs_table(agent)
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute(
            "INSERT INTO playbook_runs (agent_name, playbook_name, status, started_at) VALUES (%s, %s, 'running', NOW())",
            (agent, pb_name),
        )
        run_id = cur.lastrowid
        await cnx.commit()
        await cur.close(); await cnx.close()

        nid_map = {n["id"]: n for n in nodes if isinstance(n, dict) and "id" in n}
        visited = set()
        heads = set(nid_map.keys())
        for c in conns:
            if isinstance(c, dict):
                heads.discard(c.get("to"))
        queue = list(heads) if heads else (list(nid_map.keys())[:1])

        async def do_action(node):
            t = node.get("type", "")
            if t == "action":
                data = node.get("data") or {}
                action_name = data.get("action", "")
                params = data.get("params") or {}
                target_val = params.get("target") or (next(iter(params.values())) if params else "")
                target_for_call = _shape_soar_target(action_name, target_val)
                r = await call_agent_soar(agent, action_name, target_for_call, comment=f"playbook#{playbook_id}:{node['id']}")
                timeline.append({"node": node["id"], "type": action_name, "result": r})
                return

            cfg = node.get("config") or {}
            if t == "action.soar.block_ip":
                ip = str(cfg.get("ip", "")).strip()
                r = await call_agent_soar(agent, "block_ip", ip, comment=f"playbook#{playbook_id}:{node['id']}")
                timeline.append({"node": node["id"], "type": t, "result": r})
            elif t == "action.soar.disable_user":
                u = str(cfg.get("username", "")).strip()
                r = await call_agent_soar(agent, "disable_user", u, comment=f"playbook#{playbook_id}:{node['id']}")
                timeline.append({"node": node["id"], "type": t, "result": r})
            elif t.startswith("action.soar."):
                action_name = t.split(".")[-1]
                tgt_raw = (
                    cfg.get("target") or cfg.get("username") or cfg.get("ip") or
                    cfg.get("pid") or cfg.get("service") or cfg.get("machine") or
                    cfg.get("filepath") or cfg.get("logfile") or cfg.get("command") or ""
                )
                r = await call_agent_soar(agent, action_name, str(tgt_raw).strip(), comment=f"playbook#{playbook_id}:{node['id']}")
                timeline.append({"node": node["id"], "type": t, "result": r})
            else:
                timeline.append({"node": node["id"], "type": t, "result": "ok"})

        while queue:
            cur_id = queue.pop(0)
            if cur_id in visited or cur_id not in nid_map:
                continue
            visited.add(cur_id)
            try:
                await do_action(nid_map[cur_id])
            except Exception as node_err:
                timeline.append({
                    "node": cur_id,
                    "type": nid_map[cur_id].get("type", ""),
                    "result": {"ok": False, "error": str(node_err)},
                })
                print(f"[!] playbook node {cur_id} failed: {node_err}", flush=True)
            for nxt in nexts.get(cur_id, []):
                if nxt not in visited:
                    queue.append(nxt)

        def _step_ok(step):
            r = step.get("result")
            if isinstance(r, dict):
                return bool(r.get("ok"))
            return r == "ok" or r is True

        overall_status = "success" if (timeline and all(_step_ok(s) for s in timeline)) else "failed"
        last_err = None
        if overall_status == "failed":
            for s in timeline:
                r = s.get("result")
                if isinstance(r, dict) and not r.get("ok"):
                    last_err = r.get("error") or r.get("message")
                    if last_err:
                        break

        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        timeline_json = pyjson.dumps(timeline, ensure_ascii=False, cls=CustomEncoder)
        await cur.execute(
            "UPDATE playbook_runs SET status=%s, finished_at=NOW(), timeline=%s, last_error=%s WHERE id=%s",
            (overall_status, timeline_json, last_err, run_id),
        )
        await cnx.commit()
        await cur.close(); await cnx.close()

        return sanic_json({"ok": True, "run_id": run_id, "status": overall_status, "timeline": timeline})

    except Exception as e:
        err_msg = str(e)
        print(f"[!] run_playbook failed agent={agent} pb={playbook_id}: {err_msg}", flush=True)
        print(_tb.format_exc(), flush=True)
        if run_id is not None:
            try:
                cnx = await connect_db_for_agent(agent)
                cur = await cnx.cursor()
                tjson = pyjson.dumps(timeline + [{"error": err_msg}], ensure_ascii=False, cls=CustomEncoder)
                await cur.execute(
                    "UPDATE playbook_runs SET status='failed', finished_at=NOW(), timeline=%s WHERE id=%s",
                    (tjson, run_id),
                )
                await cnx.commit()
                await cur.close(); await cnx.close()
            except Exception as upd_err:
                print(f"[!] could not mark run {run_id} failed: {upd_err}", flush=True)
        return sanic_json({"ok": False, "error": err_msg, "run_id": run_id}, status=500)


@app.post("/<agent_name>/create_playbooks")
@require_permission("manage_soar")
async def create_playbook_compat(request, agent_name):
    data = request.json or {}
    name = (data.get("name") or "").strip()
    nodes = data.get("nodes") or []
    connections = data.get("connections") or []

    if not name:
        return sanic_json({"status": "error", "message": "name is required"}, status=400)

    await ensure_playbooks_table(agent_name)

    nodes_json = pyjson.dumps(nodes, ensure_ascii=False)
    conns_json = pyjson.dumps(connections, ensure_ascii=False)

    try:
        cnx = await connect_db_for_agent(agent_name)
        cur = await cnx.cursor()

        await cur.execute("SELECT id FROM playbooks WHERE name=%s LIMIT 1", (name,))
        row = await cur.fetchone()
        if row:
            await cur.close(); await cnx.close()
            return sanic_json(
                {"status": "error", "message": "playbook already exists for this agent"},
                status=409,
            )

        await cur.execute(
            """
            INSERT INTO playbooks (name, nodes, connections, created_at, updated_at)
            VALUES (%s, %s, %s, NOW(), NOW())
            """,
            (name, nodes_json, conns_json),
        )
        new_id = cur.lastrowid
        await cnx.commit()
        await cur.close(); await cnx.close()

        return sanic_json(
            {
                "status": "ok",
                "id": int(new_id),
                "agent_name": agent_name,
                "mode": "created",
            }
        )
    except Exception as e:
        return sanic_json(
            {"status": "error", "message": f"DB error: {e}"},
            status=500,
        )



@app.post("/<agent>/playbooks/runs/clear")
@require_permission("manage_soar")
async def clear_playbook_runs(request, agent):
    await _ensure_playbook_runs_table(agent)
    try:
        cnx = await connect_db_for_agent(agent)
        cur = await cnx.cursor()
        await cur.execute("TRUNCATE TABLE playbook_runs")
        await cnx.commit()
        await cur.close(); await cnx.close()
        return sanic_json({"status": "success", "message": "playbook_runs truncated"})
    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)



@require_permission("manage_agent")
@app.get("/<agent>/playbooks/runs/<run_id:int>")
async def get_playbook_run(request, agent, run_id):
    await _ensure_playbook_runs_table(agent)
    cnx = await connect_db_for_agent(agent)
    cur = await cnx.cursor()
    await cur.execute(
        "SELECT id, playbook_name, status, started_at, finished_at, timeline FROM playbook_runs WHERE id=%s",
        (int(run_id),)
    )
    row = await cur.fetchone()
    await cur.close(); await cnx.close()

    if not row:
        return sanic_json({"status": "error", "message": "run not found"}, status=404)

    cols = ["id", "playbook_name", "status", "started_at", "finished_at", "timeline"]
    d = dict(zip(cols, row))

    tl = d.get("timeline")
    if isinstance(tl, (bytes, bytearray)):
        tl = tl.decode("utf-8", "ignore")
    if isinstance(tl, str):
        try:
            d["timeline"] = pyjson.loads(tl)
        except Exception:
            pass

    return sanic_json(d, dumps=lambda obj: pyjson.dumps(obj, ensure_ascii=False, cls=CustomEncoder))
  

@require_permission("manage_agent")
@require_permission("manage_soar")
@app.post("/<agent>/soar/execute")
async def soar_execute(request, agent):

    if not is_soar_enabled():
        return _soar_block_response()
    data = request.json or {}
    action  = (data.get("action") or "").strip()
    target  = (data.get("target") or "").strip()
    comment = (data.get("comment") or "").strip()
    event_id = data.get("event_id")
    ttl = data.get("ttl")

    if action not in AUTOMATION_ALLOWED_ACTIONS:
        return sanic_json({"ok": False, "error": f"action not allowed: {action}"}, status=400)

    if action == ActionType.BLOCK_IP.value and not _is_valid_ipv4(target):
        return sanic_json({"ok": False, "error": "invalid IPv4"}, status=400)
    if action == ActionType.DISABLE_USER.value and not _is_valid_username(target):
        return sanic_json({"ok": False, "error": "invalid username"}, status=400)

    try:
        res = await call_agent_soar(
            agent, action=action, target=target,
            comment=comment, event_id=event_id, ttl=ttl
        )
        return sanic_json(res, status=(200 if res.get("ok") else 502))
    except Exception as e:
        return sanic_json({"ok": False, "error": str(e)}, status=500)


@require_permission("manage_db")
@app.route("/databases/<db_name>", methods=["DELETE"])
async def drop_database(request, db_name):

    try:
        if not db_name:
            return sanic_json({"status": "error", "message": "db_name cannot be empty"}, status=400)

        raw = str(db_name)
        safe = re.sub(r"[^A-Za-z0-9_]", "_", raw).strip("_")
        if not safe:
            return sanic_json({"status": "error", "message": "Invalid db_name"}, status=400)

        candidates = {safe}
        if not safe.endswith("_db"):
            candidates.add(f"{safe}_db")

        def _drop():
            with sync_mysql_conn() as conn:
                cursor = conn.cursor()
                try:
                    for name in candidates:
                        sql = f"DROP DATABASE IF EXISTS {_quote_identifier(name)}"
                        print(f"[*] Dropping candidate DB: {sql}")
                        cursor.execute(sql)
                    conn.commit()
                finally:
                    cursor.close()
        await asyncio.to_thread(_drop)

        await audit_log(request, "DROP_DATABASE", db_name, f"Databases dropped: {', '.join(candidates)}")

        return sanic_json(
            {
                "status": "success",
                "message": "Databases dropped (if existed).",
                "dropped_candidates": list(candidates),
            }
        )

    except Exception as e:
        return sanic_json({"status": "error", "message": str(e)}, status=500)


def _agent_name_forms(agent: str) -> tuple[str, str, set]:
    """Every spelling of one agent, because the platform uses two.

    `agent_identities.agent_name` stores `DESKTOP-EVS8H9J-4` while the
    database is `DESKTOP_EVS8H9J_4_db`. Deleting by one spelling leaves the
    other behind, which is how a "deleted" agent keeps appearing in the fleet
    view with its telemetry still on disk.

    Returns (hyphen form, underscore form, candidate database names).
    """
    safe = re.sub(r"[^A-Za-z0-9_-]", "", str(agent or "")).strip("-_")
    hyphen = safe.replace("_", "-")
    underscore = safe.replace("-", "_")
    dbs = set()
    for form in (safe, hyphen, underscore):
        if not form:
            continue
        base = form[:-3] if form.endswith("_db") else form
        dbs.add(f"{base}_db")
    return hyphen, underscore, dbs


@require_permission("manage_agent")
@app.route("/api/agents/<agent>", methods=["DELETE"])
async def delete_agent(request, agent):
    """Remove an agent: its telemetry database and its enrolment identity.

    Three agent databases were left behind by the installer minting a new
    identity on every re-run, and nothing in the platform could remove them.
    The fleet view counted one host four times and there was no way to say so.

    Irreversible, so it requires the agent name repeated in the body. That is
    not ceremony: the destructive half is `DROP DATABASE`, and the operator
    picking the wrong row in a list of near-identical names
    (`DESKTOP-EVS8H9J`, `-2`, `-3`, `-4`) is the realistic mistake.

    `enrollment_tokens` rows are kept. They record who enrolled what and when,
    which is audit history that outlives the agent; only `used_by_agent` is
    left pointing at a name that no longer exists, which is accurate.
    """
    hyphen, underscore, db_candidates = _agent_name_forms(agent)
    if not hyphen:
        return sanic_json({"status": "error", "message": "Invalid agent name"},
                          status=400)

    body = request.json or {}
    if str(body.get("confirm", "")).strip() not in (hyphen, underscore, str(agent)):
        return sanic_json({
            "status": "error",
            "message": "Send {\"confirm\": \"<agent name>\"} to delete. "
                       "This drops the agent's telemetry database and cannot "
                       "be undone.",
        }, status=400)

    def _purge():
        dropped, identities = [], 0
        with sync_mysql_conn() as conn:
            cursor = conn.cursor()
            try:
                for name in sorted(db_candidates):
                    cursor.execute(
                        "SELECT COUNT(*) FROM information_schema.schemata "
                        "WHERE schema_name = %s", (name,))
                    if cursor.fetchone()[0]:
                        # Identifier, so it cannot be parameterised. The name
                        # came through _agent_name_forms, which strips
                        # everything outside [A-Za-z0-9_-].
                        cursor.execute(f"DROP DATABASE {_quote_identifier(name)}")
                        dropped.append(name)
                cursor.execute(
                    "DELETE FROM userdb.agent_identities WHERE agent_name IN (%s, %s)",
                    (hyphen, underscore))
                identities = cursor.rowcount
                conn.commit()
            finally:
                cursor.close()
        return dropped, identities

    try:
        dropped, identities = await asyncio.to_thread(_purge)
    except Exception as e:
        print(f"[!] delete_agent {agent} failed: {e}", flush=True)
        return sanic_json({"status": "error", "message": str(e)}, status=500)

    await audit_log(request, "DELETE_AGENT", hyphen,
                    f"databases={dropped or 'none'} identities={identities}")

    if not dropped and not identities:
        return sanic_json({
            "status": "error",
            "message": f"No agent named {hyphen} found.",
        }, status=404)

    return sanic_json({
        "status": "success",
        "agent": hyphen,
        "databases_dropped": dropped,
        "identities_removed": identities,
    })


async def _platform_event(kind: str, *, subject: str = "",
                          source_ip: str = "", detail: str = "") -> None:
    """Raise one event about an attack on this platform, and never fail.

    Swallowed on purpose, and this is the one place in the file where that is
    right: every caller is in the middle of refusing something. A server that
    will not reject a bad login because it could not journal the rejection has
    turned a monitoring gap into an outage.
    """
    def _write():
        try:
            with sync_mysql_conn("userdb") as conn:
                cursor = conn.cursor()
                try:
                    self_defence.record(cursor, kind, subject=subject,
                                        source_ip=source_ip, detail=detail)
                    conn.commit()
                finally:
                    cursor.close()
        except Exception as e:
            print(f"[self-defence] could not record {kind}: {e}", flush=True)

    await asyncio.to_thread(_write)


AGENT_LINKS = agent_link.LinkRegistry()


@app.websocket("/agent-link")
async def agent_link_socket(request, ws):
    """A channel the agent opens and the server answers over.

    The inverse of every other server-to-agent path here, and the point of it
    is in `core.agent_link`: a host that can reach the server can be managed,
    with no port opened on the endpoint and no address to guess.

    Authenticated by `X-Agent-Key` like every other agent route. The
    fleet-wide secret is refused: this channel carries `/self_destruct` among
    other things, and a leaked master key should not be able to open one
    against every endpoint at once.
    """
    agent = await asyncio.to_thread(_validate_agent_auth_sync, request)
    if not agent or agent == "*":
        # Recorded, not only printed. A container log is where this went, and
        # nobody reads a container log until they already suspect something -
        # which is the wrong order for the one signal that says an agent key
        # is being used after it was revoked.
        kind = ("FLEET_KEY_ON_CHANNEL" if agent == "*"
                else "AGENT_KEY_REJECTED")
        print(f"[agent-link] refused a connection from {_client_ip(request)}",
              flush=True)
        await _platform_event(kind, source_ip=_client_ip(request),
                              detail="a channel was opened with a key this "
                                     "server will not accept")
        await ws.close(code=1008, reason="unauthorized")
        return

    async def send(frame) -> None:
        """Control as JSON text, stream payloads as binary.

        This serialised everything, so `json.dumps` was handed raw bytes the
        moment anything wrote *to* a stream and raised - which the relay
        caught as "the browser hung up". Keystrokes never left the server, and
        because the browser echoes its own line in pipe mode, the console
        looked alive while nothing had reached the shell.
        """
        if isinstance(frame, (bytes, bytearray)):
            await ws.send(bytes(frame))
        else:
            await ws.send(pyjson.dumps(frame))

    # The version rides the handshake, so it is known before the link is
    # registered rather than after a frame that may never arrive. An agent old
    # enough to send no header is itself the answer to "how old is it".
    reported = (request.headers.get("X-Agent-Version") or "").strip()
    state, detail = product_version.agent_support(reported)
    await _record_agent_version(agent, reported if
                                product_version.is_valid_version(reported) else None)

    link = agent_link.AgentLink(agent, send)
    replaced = AGENT_LINKS.register(link)
    if replaced is not None:
        # Usually a drop the agent noticed before we did. Failing whatever was
        # in flight on the old socket beats leaving it to time out.
        replaced.close("replaced by a new connection from the same agent")
    print(f"[agent-link] {agent} connected from {_client_ip(request)} "
          f"[{state}: {detail}]", flush=True)
    if state == "unsupported":
        # Loud, and still connected. Turning it away would leave it
        # permanently unreachable and unupgradeable, because there is no
        # second route to an endpoint any more.
        print(f"[agent-link] {agent} is running an unsupported agent. "
              f"Rebuild and reinstall it.", flush=True)
    await audit_log(request, "AGENT_LINK_OPEN", agent,
                    f"agent opened a channel ({state}: {detail})")

    reason = "the agent disconnected"
    #: Channels we have already told this agent to stop. Frames sent before
    #: that close arrived are ordinary, and one line each would be the
    #: noisiest thing in the log at exactly the wrong moment.
    orphaned: set[int] = set()
    try:
        while True:
            raw = await ws.recv()
            if raw is None:
                break

            # Binary is a stream payload; the socket keeps the two apart.
            if isinstance(raw, (bytes, bytearray)):
                parsed = agent_link.decode_stream_frame(raw)
                if parsed is None:
                    link.touch()
                    continue
                channel, payload = parsed
                if not link.deliver_stream(channel, payload):
                    # Say so, rather than only noticing. This printed a line
                    # per frame and changed nothing, so an agent whose viewer
                    # had gone kept capturing and sending for the life of the
                    # connection - and every one of those frames competed for
                    # the agent's single send lock with the config reads and
                    # restarts queued behind them. That is how a healthy,
                    # connected agent came to answer GET /config/rules in
                    # more than five seconds.
                    #
                    # Once per channel: the close is already travelling, and
                    # the frames still in flight were sent before it arrived.
                    if channel not in orphaned:
                        orphaned.add(channel)
                        print(f"[agent-link] {agent}: data for a stream that "
                              f"is gone (ch {channel}); telling it to stop",
                              flush=True)
                        await _ws_notify(ws, pyjson.dumps({
                            "t": "close", "ch": channel,
                            "why": "nothing is attached to this stream",
                        }))
                continue

            try:
                frame = pyjson.loads(raw)
            except Exception:
                # An endpoint sending noise must not take the channel down for
                # itself; the next frame may well be fine.
                link.touch()
                continue
            if not isinstance(frame, dict):
                link.touch()
                continue

            kind = frame.get("t")
            if kind == "res":
                if not link.deliver(frame):
                    print(f"[agent-link] {agent}: reply to a request nobody is "
                          f"waiting for ({frame.get('id')})", flush=True)
            elif kind == "stream_opened":
                link.stream_opened(frame.get("ch"))
            elif kind == "stream_failed":
                link.stream_failed(frame.get("ch"),
                                   str(frame.get("why") or "the agent refused"))
            elif kind == "close":
                link.close_stream(frame.get("ch"),
                                  str(frame.get("why") or "the agent closed it"))
            elif kind == "ping":
                link.touch()
                await _ws_notify(ws, pyjson.dumps({"t": "pong"}))
            else:
                link.touch()
    except Exception as e:
        reason = f"the channel failed: {e}"
    finally:
        link.close(reason)
        AGENT_LINKS.unregister(link)
        print(f"[agent-link] {agent} disconnected: {reason}", flush=True)


def _validate_agent_auth_sync(request) -> str | None:
    """Resolve an agent from its key, off the event loop.

    A synchronous MySQL connection on a websocket handshake blocks every other
    request on this worker for its duration, which is how one reconnecting
    agent slows the console for everyone. The caller runs this in a thread.

    Deliberately narrower than `_validate_agent_auth`: no fallback to the
    fleet-wide secret. This channel carries `/self_destruct`, and a leaked
    master key should not open one against every endpoint at once.
    """
    key = request.headers.get("X-Agent-Key") or request.args.get("agent_key")
    if not key:
        return None
    try:
        with sync_mysql_conn("userdb") as conn:
            cur = conn.cursor()
            try:
                cur.execute(
                    "SELECT agent_name FROM agent_identities "
                    "WHERE agent_key=%s AND revoked_at IS NULL",
                    (key,))
                row = cur.fetchone()
                return row[0] if row else None
            finally:
                cur.close()
    except Exception as e:
        print(f"[agent-link] identity lookup failed: {e}", flush=True)
        return None


async def _ws_notify(ws, payload: str) -> None:
    """Tell the browser something, if it is still listening.

    The one swallow on the console path. Every caller is about to close the
    socket and has already logged the reason, so a browser that has gone away
    cannot be told anything. Separate try/excepts saying the same thing is how
    a real error ends up hidden among them.
    """
    try:
        await ws.send(payload)
    except Exception:
        pass


@app.websocket("/console-proxy/<agent>")
@require_permission("manage_soar")
async def console_proxy(request, ws, agent):
    """Browser <-> agent interactive shell.

    `manage_soar`, not `read_telemetry`. The screen stream is an observation
    and is gated as one; this is a root shell on the endpoint. The permission
    that already governs `run_cmd` is the one that governs this, because they
    are the same power.

    Audited on open rather than on close: a session that ends because the
    server died would otherwise leave no record that it happened at all.
    """
    await audit_log(request, "CONSOLE_OPEN", agent,
                    f"interactive shell requested by {_client_ip(request)}")

    link = _linked_agent(agent)
    if link is None:
        await _ws_notify(ws, pyjson.dumps(
            {"t": "e", "d": _no_channel_message(agent)}))
        await ws.close()
        return

    try:
        await _relay_channel_stream(ws, link, "console", {})
        await audit_log(request, "CONSOLE_CLOSE", agent, "via the channel")
    except agent_link.LinkError as e:
        # The agent's own sentence, not a summary of it. It says why it would
        # not open a console - a shell already running, no PTY available, a
        # helper that could not reach the desktop - and while a direct
        # fallback existed that reason was replaced by the fallback's
        # "Connection refused", which described the second attempt and hid
        # the first.
        print(f"[console-proxy] {agent}: channel console failed ({e})", flush=True)
        await _ws_notify(ws, pyjson.dumps(
            {"t": "e", "d": f"the agent would not open a console: {e}"}))
        await ws.close()


async def _relay_channel_stream(ws, link, kind: str, args: dict,
                                *, binary: bool = False) -> None:
    """Pump a browser websocket against a stream on the agent's channel.

    The same shape as the direct relays below - two pumps and a gather - so
    the console and the screen behave identically whichever transport carried
    them, and a reader comparing the two is not comparing two designs.
    """
    stream = await link.open_stream(kind, args)

    async def to_agent():
        try:
            while True:
                data = await ws.recv()
                if data is None:
                    return
                if isinstance(data, str):
                    data = data.encode("utf-8", "replace")
                await stream.send(data)
        except Exception:
            # The browser hung up, which is the ordinary end of a console.
            return

    async def to_browser():
        try:
            while True:
                payload = await stream.receive()
                if payload is None:
                    return
                if not payload:
                    continue
                # A console is text and a screen is a JPEG. Sending a frame
                # as text would corrupt it on the way through, and the
                # browser would render a broken image rather than say why.
                await ws.send(payload if binary
                              else payload.decode("utf-8", "replace"))
        except Exception:
            return

    try:
        await asyncio.gather(to_agent(), to_browser())
    finally:
        # `end_stream`, not `close_stream`: the latter only forgets it here.
        # A stream left open on the agent is a shell running with nobody
        # attached - and since a console is one per host, the next request is
        # then refused as a duplicate of a session nobody can reach.
        await link.end_stream(stream.channel, "the viewer went away")


@app.websocket("/vnc-proxy/<agent>")
@require_permission("read_telemetry")
async def vnc_proxy(request, ws, agent):
    """Browser <-> agent screen relay.

    Registered at module scope (was previously nested under `if __name__ ==
    "__main__":`) so it survives Sanic worker subprocesses and any embedding
    that imports app.py instead of running it as the main script.
    """
    fps = request.args.get("fps", "10")
    q = request.args.get("q", "60")
    w = request.args.get("w", "1280")

    link = _linked_agent(agent)
    if link is None:
        await _ws_notify(ws, pyjson.dumps(
            {"error": "agent_unreachable", "detail": _no_channel_message(agent)}))
        await ws.close()
        return

    try:
        await _relay_channel_stream(
            ws, link, "screen", {"fps": fps, "q": q, "w": w}, binary=True)
    except agent_link.LinkError as e:
        print(f"[screen-proxy] {agent}: channel screen failed ({e})", flush=True)
        await _ws_notify(ws, pyjson.dumps(
            {"error": "agent_channel", "detail": str(e)}))
        await ws.close()


@app.route("/", name="frontend_root")
async def serve_root(request):
    return await response.file("./frontend/dist/index.html")

#: Page size. Capped because `size` reaches OpenSearch directly, and a
#: request for a million rows is answered by building a million rows.
SEARCH_MAX_SIZE = 200
SEARCH_DEFAULT_SIZE = 50

#: How far back a search looks when nothing says otherwise.
#:
#: A default at all, rather than "everything", is the single biggest change
#: here. The previous version had no time filter of any kind, so every search
#: scanned the whole retention and sorted it - and no operator could ask the
#: question they actually had, which is nearly always about a window.
SEARCH_DEFAULT_WINDOW = "now-24h"

_RELATIVE_TIME = re.compile(r"^now(-\d+[smhdwMy])?(/[smhdwMy])?$")


def _time_bound(value: str | None, fallback: str) -> str:
    """An OpenSearch date-math bound, or the fallback.

    `now-7d` and an ISO timestamp are both accepted; anything else is
    refused rather than passed through, because an unparseable bound sent to
    the engine comes back as a 400 the operator has to decode.
    """
    text = (value or "").strip()
    if not text:
        return fallback
    if _RELATIVE_TIME.match(text):
        return text
    try:
        datetime.fromisoformat(text.replace("Z", "+00:00"))
        return text
    except ValueError:
        return fallback


def _positive_int(value, default: int, maximum: int) -> int:
    """A bounded integer from a query string.

    `int(request.args.get("limit", 100))` was the previous version: `?limit=abc`
    raised inside the handler and came back as a bodyless 500, and
    `?limit=999999` was honoured.
    """
    try:
        parsed = int(str(value))
    except (TypeError, ValueError):
        return default
    return max(1, min(parsed, maximum))


@require_permission("read_telemetry")
@app.route("/api/logs/search")
async def search_logs_api(request):
    """Search indexed telemetry: a window, a query, and a page of results.

    What this replaces could do one thing - free text across every field, over
    the whole retention, first N results, no way to see the N+1th. Three of
    those are what an operator asks for first.

    The query is Lucene syntax (`severity:critical AND source:sshd`), which is
    what the filter builder in the console writes and what somebody who knows
    the syntax can edit by hand. Leading wildcards are refused: `*foo` forces
    a scan of every term in the index and is the classic way to take a search
    cluster down from a text box.
    """
    query_text = (request.args.get("q") or "").strip()
    agent = (request.args.get("agent") or "").strip()
    table = (request.args.get("table") or "").strip()
    size = _positive_int(request.args.get("size"),
                         SEARCH_DEFAULT_SIZE, SEARCH_MAX_SIZE)
    page = _positive_int(request.args.get("page"), 1, 10_000) - 1

    filters: list = [{
        "range": {"@timestamp": {
            "gte": _time_bound(request.args.get("from"), SEARCH_DEFAULT_WINDOW),
            "lte": _time_bound(request.args.get("to"), "now"),
        }},
    }]
    if agent:
        # An exact term, not a wildcard. The previous version put the caller's
        # string straight into a `wildcard` clause, so the default `*` matched
        # every term in the index and a leading `*` was one keystroke away.
        hyphen, underscore, _ = _agent_name_forms(agent)
        filters.append({"terms": {"agent_name.keyword":
                                  sorted({agent, hyphen, underscore})}})

    must: list = []
    if query_text:
        must.append({"query_string": {
            "query": query_text,
            "default_operator": "AND",
            "analyze_wildcard": True,
            "allow_leading_wildcard": False,
            "lenient": True,
        }})

    body = {
        "from": page * size,
        "size": size,
        "track_total_hits": True,
        "query": {"bool": {"must": must or [{"match_all": {}}], "filter": filters}},
        "sort": [{"@timestamp": {"order": "desc"}}],
    }

    index_mask = (f"sentora-logs-{table.replace('_', '-')}"
                  if table else "sentora-logs-*")
    try:
        resp = await asyncio.to_thread(
            os_utils.search_logs, body, index_mask)
    except os_utils.SearchError as e:
        # 400, not an empty result. A search that could not run and a search
        # that found nothing were the same screen, and on a security console
        # one of those is an all-clear and the other is no answer at all.
        return sanic_json({
            "status": "error",
            "message": f"The query could not be run: {e}",
            "hits": [], "total": 0,
        }, status=400)

    hits = resp.get("hits", {})
    total = (hits.get("total") or {}).get("value", 0)
    return sanic_json({
        "status": "success",
        "hits": [{**h.get("_source", {}), "_index": h.get("_index")}
                 for h in hits.get("hits", [])],
        "total": total,
        "page": page + 1,
        "size": size,
        "pages": max(1, -(-total // size)),
        "took_ms": resp.get("took"),
    })


#: Columns the Events search may filter and sort on, per table.
#:
#: An allowlist, because these become SQL identifiers. Everything the operator
#: types goes in as a parameter; only the *column* comes from here.
EVENT_SEARCH_TABLES = {
    "siem_events": ("id", "source", "severity", "message", "techniques",
                    "created_at"),
    "events_alert": ("id", "source", "severity", "message", "created_at"),
}

EVENT_SEARCH_MAX_SIZE = 200


def _fernet() -> "Fernet":
    """The key this server decrypts telemetry with."""
    key = getattr(app.ctx, "fernet_key", None) or bootstrap_client.get_fernet_key()
    return Fernet(key.encode("utf-8") if isinstance(key, str) else key)


def _search_events_sync(table: str, term: str, agent: str | None,
                        severity: str | None, hours: int,
                        page: int, size: int) -> tuple[list, int]:
    """Search the per-agent MySQL tables. Returns (rows, total).

    Separate from the log search rather than merged with it. The two stores
    answer different questions - OpenSearch holds raw telemetry, these hold
    what the rules decided about it - and merging them means one ranking over
    two things that are not comparable. Tabs, not a union.
    """
    columns = EVENT_SEARCH_TABLES[table]
    encrypted = set(ENCRYPTED_FIELDS_MAP.get(table) or ())

    rows: list = []
    total = 0
    with sync_mysql_conn() as conn:
        cursor = conn.cursor(dictionary=True)
        try:
            cursor.execute("SHOW DATABASES")
            dbs = [r["Database"] for r in cursor.fetchall()
                   if str(r["Database"]).endswith("_db")]
            if agent:
                wanted = _agent_name_forms(agent)[1] + "_db"
                dbs = [d for d in dbs if d == wanted]

            for db in dbs:
                where = ["created_at >= NOW() - INTERVAL %s HOUR"]
                params: list = [hours]
                # `severity` is encrypted in some tables, so it cannot be
                # filtered in SQL there - it is applied after decryption below.
                if severity and "severity" in columns and "severity" not in encrypted:
                    where.append("severity = %s")
                    params.append(severity)
                clause = " AND ".join(where)
                # Through the escaper even though `columns` is an allowlist and
                # `db` came from SHOW DATABASES. Relying on where a name came
                # from is how the rule rots: the next person adds one that came
                # from somewhere else and the shape looks identical.
                select = ", ".join(_quote_identifier(c) for c in columns)
                try:
                    cursor.execute(
                        f"SELECT {select} FROM {_quote_identifier(db)}"
                        f".{_quote_identifier(table)} "
                        f"WHERE {clause} ORDER BY created_at DESC LIMIT 2000",
                        params)
                except Exception:
                    continue                # table not provisioned on this agent
                for row in cursor.fetchall():
                    row["agent"] = db[:-3]
                    rows.append(row)
        finally:
            cursor.close()

    # Decrypt, then filter. The searchable text is inside the ciphertext, so
    # a `LIKE` in SQL would match nothing and report it as no results - which
    # is the failure this whole page is being rebuilt around.
    if encrypted:
        fernet_obj = _fernet()
        rows = [decrypt_row_fields(r, list(encrypted), fernet_obj) for r in rows]

    if term:
        needle = term.lower()
        rows = [r for r in rows
                if any(needle in str(v).lower() for v in r.values())]
    if severity and "severity" in encrypted:
        rows = [r for r in rows
                if str(r.get("severity") or "").lower() == severity.lower()]

    rows.sort(key=lambda r: str(r.get("created_at") or ""), reverse=True)
    total = len(rows)
    start = (page - 1) * size
    return rows[start:start + size], total


@require_permission("read_telemetry")
@app.route("/api/events/search")
async def search_events_api(request):
    """Search what the rules decided, as opposed to the raw telemetry.

    The Events half of the search page. Kept as its own endpoint and its own
    tab because the two stores are not comparable: one ranking over both would
    have to decide whether a raw log line outranks a detection, and there is
    no answer to that.
    """
    table = (request.args.get("table") or "siem_events").strip()
    if table not in EVENT_SEARCH_TABLES:
        return sanic_json({
            "status": "error",
            "message": f"{table!r} is not searchable; "
                       f"try {sorted(EVENT_SEARCH_TABLES)}",
        }, status=400)

    size = _positive_int(request.args.get("size"), 50, EVENT_SEARCH_MAX_SIZE)
    page = _positive_int(request.args.get("page"), 1, 10_000)
    hours = _positive_int(request.args.get("hours"), 24, 24 * 365)

    try:
        rows, total = await asyncio.to_thread(
            _search_events_sync, table,
            (request.args.get("q") or "").strip(),
            (request.args.get("agent") or "").strip() or None,
            (request.args.get("severity") or "").strip() or None,
            hours, page, size)
    except Exception as e:
        print(f"[!] event search failed: {e}", flush=True)
        return sanic_json({"status": "error",
                           "message": f"The search could not be run: {e}"},
                          status=500)

    return sanic_json({
        "status": "success",
        "hits": rows,
        "total": total,
        "page": page,
        "size": size,
        "pages": max(1, -(-total // size)),
        "tables": sorted(EVENT_SEARCH_TABLES),
    })


@require_permission("read_telemetry")
@app.route("/api/logs/fields")
async def search_log_fields(request):
    """The field names the filter builder offers.

    Read from the index mapping rather than a hand-kept list: the indices are
    created by dynamic mapping from whatever the agents send, so a list here
    would describe a schema nobody wrote.
    """
    fields = await asyncio.to_thread(os_utils.log_fields)
    return sanic_json({"status": "success", "fields": fields})

@app.route("/<path:path>", name="frontend_spa")
async def serve_index(request, path=""):
    if path.startswith(("api/", "health", "assets/", "vite.svg")):
         return sanic_json({"error": "Not Found", "path": path}, status=404)
         
    return await response.file("./frontend/dist/index.html")

@app.route("/health")
async def health_check(request):
    """Liveness, and what is running.

    The version is here because this is the one endpoint that answers without
    a session, which makes it the only thing a deploy script or a load
    balancer can ask. `current_agent_version` is what a fleet should be on;
    the installer reads it so a freshly generated deployment does not have to
    be told separately.
    """
    return sanic_json({
        "status": "healthy",
        "timestamp": datetime.now().isoformat(),
        "server_version": product_version.SERVER_VERSION,
        "current_agent_version": product_version.CURRENT_AGENT_VERSION,
        "min_agent_version": product_version.MIN_AGENT_VERSION,
    })

if __name__ == "__main__":
    app.config.AUTO_RELOAD = False
    app.config.TOUCHUP = False
    app.config.ACCESS_LOG = True
    
    # `WORKERS` was in docker-compose.yaml and was never read: this line took
    # `cpu_count()` regardless, so a deployment whose configuration said one
    # worker ran eight. Reading it changes nothing for anyone who has not set
    # it, and does what the compose file has always said.
    #
    # It matters beyond tidiness now. Each worker is a separate process with
    # its own state, so anything an agent connects *to* - see
    # `core.agent_link` - exists in one worker and is invisible to the rest.
    # A request that happened to land on another worker would silently take a
    # different path, which is the hardest kind of bug to be handed.
    configured = os.getenv("WORKERS", "").strip()
    if configured.isdigit() and int(configured) > 0:
        num_workers = int(configured)
    else:
        num_workers = multiprocessing.cpu_count() if os.name != 'nt' else 1

    if num_workers > 1:
        print(f"[!] Running {num_workers} workers. Agent-initiated channels "
              f"live in one worker only; set WORKERS=1 until the registry is "
              f"shared between them.", flush=True)

    # TLS_ENABLED / TLS_CERT / TLS_KEY were documented in .env.example, in
    # docs/production-deployment.md and in the output of
    # certs/generate_certs.py, and read by nothing. Setting them produced a
    # console that came up cleanly on plain HTTP while its operator had every
    # reason to think it was HTTPS - so the login form and the session cookie
    # crossed the network in clear text with no symptom at all.
    #
    # `core.tls.resolve` raises rather than falling back. A server that cannot
    # do what its configuration says must not quietly do something weaker.
    try:
        ssl_config = product_tls.resolve()
    except product_tls.TLSConfigError as e:
        print(f"[!] {e}", flush=True)
        raise SystemExit(2)

    print(f"[*] Serving on port 8000 over {product_tls.describe()}", flush=True)
    if ssl_config and not SESSION_COOKIE_SECURE:
        # Reachable only when SESSION_COOKIE_SECURE=0 is set by hand against
        # an HTTPS listener, which is a downgrade rather than a default.
        print("[!] TLS is on but SESSION_COOKIE_SECURE=0 is set explicitly; "
              "the session cookie is allowed onto plain HTTP.", flush=True)

    app.run(
        host="0.0.0.0",
        port=8000,
        single_process=(num_workers == 1),
        workers=num_workers,
        access_log=True,
        auto_reload=False,
        ssl=ssl_config,
    )