Security keys, a table that never had a collector, and the console around them

app.py

@@ -2792,6 +2792,26 @@ async def get_network_inventory(request, agent):
     return await stream_from_db_dec("network_connections", agent,
                                     connect_db_for_agent, encrypted_fields=fields)
 
+@require_permission("read_telemetry")
+@app.route("/api/agent/<agent>/inventory/ports")
+async def get_listening_ports(request, agent):
+    # `network_inventory` was collected every ten minutes, shipped,
+    # deduplicated, stored - and read by nothing at all. It was the last table
+    # in `ALLOWED_TABLES` in that state, after `security_audit` (which had no
+    # collector) and `software_inventory` (which had no reader).
+    #
+    # It is not the same data as the Network tab above. `network_connections`
+    # is conversations already underway; this is what the host is listening
+    # on, which is the question "what can be reached from outside" - and it is
+    # a better answer than `portscan_result` gives, because a scan infers the
+    # surface from outside while this is the kernel's own list, with the owning
+    # process named.
+    #
+    # Plaintext, so no fields: nothing in it is sensitive on its own and the
+    # port has to be indexable.
+    return await stream_from_db_dec("network_inventory", agent,
+                                    connect_db_for_agent)
+
 @require_permission("read_telemetry")
 @app.route("/api/agent/<agent>/fim")
 async def get_fim_data(request, agent):

core/opensearch.py

@@ -3,6 +3,8 @@
 from datetime import datetime
 from opensearchpy import OpenSearch, RequestsHttpConnection
 
+from core import telemetry_crypto
+
 OPENSEARCH_URL = os.getenv("OPENSEARCH_URL", "http://opensearch:9200")
 OPENSEARCH_USER = os.getenv("OPENSEARCH_USER", "admin")
 OPENSEARCH_PASS = os.getenv("OPENSEARCH_PASSWORD", "admin")
@@ -15,20 +17,84 @@
     connection_class=RequestsHttpConnection
 )
 
+#: Tables whose rows are indexed for search, and the reason the rest are not.
+#:
+#: Every table used to be indexed, which was never a decision - `index_log` was
+#: called unconditionally for whatever ingest had just stored. Two costs came
+#: out of that:
+#:
+#:   - **the index held ciphertext.** `siem_events.message` went in as
+#:     `enc::gAAAA...`, so `message:powershell` matched 0 of 69 documents while
+#:     `source:PowerShell` matched 52 of the same 69. Nine indices and roughly
+#:     863,000 documents were unsearchable on the only field that mattered.
+#:   - **it grew without bound.** A snapshot table is emptied and rewritten in
+#:     MySQL on every batch; the index kept every copy. 426,768
+#:     `hardware_inventory` documents for a CPU, some RAM and four disks.
+#:
+#: The rule is what the search is *for*. An index answers "find this, anywhere
+#: in the fleet, in this window" - which suits an append-only event log and
+#: does not suit a table whose answer is "the current list for this host".
+#: State tables have a route and a view each, and a view gives an exact answer
+#: where a time-windowed full-text search gives a vague one.
+INDEXED_TABLES = frozenset({
+    "siem_events", "events_alert",        # the log, and what the rules said
+    "process_events", "registry_logs",    # EDR observations
+    "fim_data", "critical_files",         # file and permission changes
+    "security_audit",                     # posture findings, hunted fleet-wide
+    "soar_actions",                       # what was done in response
+    "portscan_result",                    # small, and "who has 3389 open" is
+                                          # exactly an index question
+    "audit_logs",                         # platform, not agent: who did what
+})
+
+#: Deliberately not indexed, with the view that answers the question instead.
+#: Kept beside the allowlist so the reason arrives with the name, and asserted
+#: against `server.ALLOWED_TABLES` by `tests/test_opensearch_indexing.py` - a
+#: table added to ingest and to neither of these is a silent decision.
+NOT_INDEXED = {
+    "hardware_inventory": "Assets > Hardware. 426,768 documents for one host's components.",
+    "software_inventory": "Assets > Software.",
+    "network_inventory": "Assets > Ports.",
+    "network_connections": "Assets > Network. A snapshot, rewritten every batch.",
+    "packages": "Input to the vulnerability scanner; surfaced as vulnerabilities_report.",
+    "docker_containers": "Its own view, and a snapshot.",
+    "disk_usage": "Dashboard metrics, not events.",
+    "resource_usage": "Dashboard metrics, not events.",
+    "vulnerabilities_report": "Written server-side by scanners/vuln.py, which never called this.",
+}
+
+
 async def index_log(agent: str, table: str, item: dict):
+    """Index one row for search, decrypted, if this table is searched at all.
+
+    **Decrypted here and nowhere upstream.** `core/telemetry_crypto` lists
+    where agent telemetry is turned back into plaintext - the prompt and the
+    correlation fields - and where it is not: the stored row and the broker
+    message. The index was a fifth destination that list never mentioned, and
+    it got ciphertext by default rather than by choice.
+
+    Indexing plaintext is a real cost, not a free fix: `sentora-logs-*` becomes
+    a readable copy of the telemetry it covers, so it has to be treated as
+    sensitive - see `docs/production-deployment.md`. The alternative was a hunt
+    surface that cannot match a process name, which is not a security control,
+    only a broken feature that reports success.
+
+    A value that will not decrypt is indexed as it stands. `decrypt_value`
+    counts those, and a missing key is reported rather than guessed at: an
+    index quietly holding ciphertext is what this change exists to end.
     """
-    Index a single log entry into OpenSearch.
-    Index name pattern: sentora-logs-<table_name>
-    """
+    if table not in INDEXED_TABLES:
+        return None
+
     try:
         index_name = f"sentora-logs-{table.replace('_', '-')}"
-        
-        doc = dict(item)
+
+        doc = telemetry_crypto.decrypt_item(table, dict(item))
         doc["agent_name"] = agent
         doc["@timestamp"] = datetime.now().isoformat()
-        
+
         doc.pop("id", None)
-        
+
         response = client.index(
             index=index_name,
             body=doc,

docs/production-deployment.md

@@ -139,6 +139,30 @@ authenticate in the default configuration: OpenSearch runs with
 log the platform has collected. Publish through the same reverse proxy and
 auth as `app` instead.
 
+**OpenSearch holds plaintext telemetry, and MySQL does not.** This is the one
+place in the stack where that is true, so it changes what a compromise of the
+volume yields:
+
+| Store | Volume | What a copy of it gives an attacker |
+| :--- | :--- | :--- |
+| MySQL | `mysql_data` | Ciphertext for the encrypted columns. Needs `data/fernet.key` as well to be readable. |
+| RabbitMQ | — | Ciphertext. A queued message is data at rest for as long as it is queued, so it is left encrypted. |
+| **OpenSearch** | `opensearch_data` | **Readable log lines, process command lines, registry values and file paths** — for the event tables it indexes. |
+
+That is a deliberate trade, not an oversight. The index used to hold
+`enc::gAAAA...` in the field operators search on, which meant
+`message:powershell` matched none of the events that contained it while
+`source:PowerShell` matched 52 — a hunting surface that reports success and
+finds nothing is not a security control. `core/opensearch.INDEXED_TABLES`
+limits the plaintext to the tables a search is actually for; the inventory and
+snapshot tables are excluded and answered by their own views.
+
+So: treat `opensearch_data` and port 9200 as you would the database, keep
+`data/fernet.key` on a different trust boundary from the index if you can, and
+run `scripts/prune_search_indices.py` once after upgrading — the indices
+written before the allowlist existed are still on disk, roughly 1.1 million
+documents of stale inventory copies.
+
 The "Open Dashboards" button in Log Explorer links to port 5601 on the
 server's hostname, so it works from the host itself; remote operators need
 that proxy.

frontend/src/pages/Assets.tsx

@@ -3,8 +3,9 @@ import {
   Database, 
   Search, 
   Cpu, 
-  Package, 
-  Globe, 
+  Package,
+  Globe,
+  EthernetPort,
   Server,
   RefreshCw,
   ChevronRight,
@@ -31,7 +32,7 @@ const STATE_TONE: Record<string, string> = {
 const Assets: React.FC = () => {
     const [agents, setAgents] = useState<any[]>([]);
     const [selectedAgent, setSelectedAgent] = useState<string | null>(null);
-    const [activeTab, setActiveTab] = useState<'hardware' | 'software' | 'network'>('hardware');
+    const [activeTab, setActiveTab] = useState<'hardware' | 'software' | 'network' | 'ports'>('hardware');
     const [data, setData] = useState<any[]>([]);
     const [loading, setLoading] = useState(false);
     const [searchTerm, setSearchTerm] = useState('');
@@ -86,6 +87,10 @@ const Assets: React.FC = () => {
         const field =
             activeTab === 'hardware' ? (i: any) => i.type
             : activeTab === 'software' ? (i: any) => i.vendor
+            // Every listening socket is in state LISTEN, so grouping ports by
+            // state draws one bar. The useful question is which process owns
+            // the surface.
+            : activeTab === 'ports' ? (i: any) => i.process_name
             : (i: any) => i.state;
 
         const counts = new Map<string, number>();
@@ -106,8 +111,11 @@ const Assets: React.FC = () => {
         ? { title: 'Hardware by type',
             note: 'What the inventory is made of. A category that is missing entirely usually means the collector for it failed rather than the host has none.' }
         : activeTab === 'software'
-        ? { title: 'Packages by vendor',
+        ? { title: 'Software by vendor',
             note: 'Who wrote the software on this host. A long tail of one-off vendors is where unmanaged software lives.' }
+        : activeTab === 'ports'
+        ? { title: 'Listening ports by process',
+            note: 'The host’s own account of what can be reached from outside, with the process that owns each socket. A listener you cannot name is the one to look at first.' }
         : { title: 'Connections by state',
             note: 'Listening sockets are attack surface; established ones are traffic already flowing. The mix is the shape of what this host is doing.' };
 
@@ -202,10 +210,15 @@ const Assets: React.FC = () => {
                 {/* Main Content */}
                 <div style={{ display: 'flex', flexDirection: 'column', gap: '24px' }}>
                     {/* Tabs */}
-                    <div style={{ display: 'flex', background: 'var(--card-bg)', padding: '6px', borderRadius: '14px', border: '1px solid var(--border-color)', width: 'fit-content' }}>
+                    {/* flexWrap, because a fourth tab is what takes this row
+                        past a narrow viewport. `width: fit-content` does not
+                        shrink below its content, so without wrapping the last
+                        tab is simply unreachable on a phone. */}
+                    <div style={{ display: 'flex', flexWrap: 'wrap', background: 'var(--card-bg)', padding: '6px', borderRadius: '14px', border: '1px solid var(--border-color)', width: 'fit-content', maxWidth: '100%' }}>
                         <TabButton active={activeTab === 'hardware'} onClick={() => setActiveTab('hardware')} icon={<Cpu size={16} />} label="Hardware" />
                         <TabButton active={activeTab === 'software'} onClick={() => setActiveTab('software')} icon={<Package size={16} />} label="Software" />
                         <TabButton active={activeTab === 'network'} onClick={() => setActiveTab('network')} icon={<Globe size={16} />} label="Network" />
+                        <TabButton active={activeTab === 'ports'} onClick={() => setActiveTab('ports')} icon={<EthernetPort size={16} />} label="Ports" />
                     </div>
 
                     <div style={{ marginBottom: '24px' }}>
@@ -262,6 +275,18 @@ const Assets: React.FC = () => {
                                             <Th>Remote</Th>
                                             <Th>State</Th>
                                         </>}
+                                        {/* No State column and no Remote:
+                                            every row here is a listener, so
+                                            both are constant. A column with
+                                            the same value on every row spends
+                                            width to say nothing. */}
+                                        {activeTab === 'ports' && <>
+                                            <Th>Port</Th>
+                                            <Th>Protocol</Th>
+                                            <Th>Bound To</Th>
+                                            <Th>Process</Th>
+                                            <Th>PID</Th>
+                                        </>}
                                     </tr>
                                 </thead>
                                 <tbody>
@@ -308,6 +333,13 @@ const Assets: React.FC = () => {
                                                 </Td>
                                                 <Td><StatusBadge status={item.state} /></Td>
                                             </>}
+                                            {activeTab === 'ports' && <>
+                                                <Td className="mono" style={{ fontWeight: 700, color: 'var(--accent-secondary)' }}>{item.local_port}</Td>
+                                                <Td style={{ opacity: 0.7 }}>{item.protocol}</Td>
+                                                <Td><Reach address={item.local_address} /></Td>
+                                                <Td style={{ fontWeight: 600 }}>{item.process_name}</Td>
+                                                <Td style={{ opacity: 0.6 }}>{item.pid}</Td>
+                                            </>}
                                         </tr>
                                     ))}
                                 </tbody>
@@ -342,6 +374,36 @@ const TabButton = ({ active, onClick, icon, label }: any) => (
     </button>
 );
 
+/** What a bind address means for reach, which is the only thing that makes one
+ *  listening port more interesting than another.
+ *
+ *  `0.0.0.0` and `::` accept from any interface; `127.0.0.1` and `::1` cannot
+ *  be reached off the host at all. Rendering both as a bare address leaves the
+ *  operator to make that distinction 80 rows at a time, and it is the whole
+ *  question they came to the tab with. */
+const LOCAL_ONLY = new Set(['127.0.0.1', '::1', 'localhost']);
+const ANY_INTERFACE = new Set(['0.0.0.0', '::', '*', '']);
+
+const Reach = ({ address }: { address: string }) => {
+    const addr = (address ?? '').trim();
+    const exposed = ANY_INTERFACE.has(addr);
+    const local = LOCAL_ONLY.has(addr);
+    return (
+        <span style={{ display: 'inline-flex', alignItems: 'baseline', gap: '8px' }}>
+            <span className="mono">{addr || '0.0.0.0'}</span>
+            <span style={{
+                fontSize: '0.7rem', fontWeight: 700, textTransform: 'uppercase',
+                letterSpacing: '0.04em',
+                color: exposed ? 'var(--sev-high)'
+                    : local ? 'var(--text-muted)'
+                    : 'var(--sev-medium)',
+            }}>
+                {exposed ? 'all interfaces' : local ? 'loopback' : 'one interface'}
+            </span>
+        </span>
+    );
+};
+
 const Th = ({ children }: any) => (
     <th style={{ padding: '16px 20px', fontSize: '0.8rem', fontWeight: 700, color: 'var(--text-secondary)', textTransform: 'uppercase', letterSpacing: '0.05em' }}>{children}</th>
 );

scripts/prune_search_indices.py

@@ -0,0 +1,98 @@
+"""Remove the search indices Sentora no longer writes.
+
+`core/opensearch.INDEXED_TABLES` is an allowlist now. Before it existed, every
+ingested table was indexed, which produced two piles nobody asked for:
+
+    sentora-logs-hardware-inventory   426,768 documents
+    sentora-logs-packages             255,534
+    sentora-logs-software-inventory   165,297
+    sentora-logs-network-connections  165,068
+    sentora-logs-docker-containers     63,364
+    sentora-logs-network-inventory     38,142
+    sentora-logs-resource-usage        26,181
+    sentora-logs-disk-usage             2,630
+
+Those are snapshot and inventory tables. MySQL empties and rewrites them on
+every batch; the index kept every copy and none of the deletes, so what looks
+like history is a pile of stale rows with no way to tell which one is current.
+Each has a route and a view that answers the question exactly.
+
+**Not run automatically, and not from the server.** Deleting an index is not
+reversible and these may be the backing store for saved visualisations in
+OpenSearch Dashboards - which this script cannot see and must not assume
+about. So it prints, asks, and only then deletes.
+
+    docker compose exec sentora-server python scripts/prune_search_indices.py
+    docker compose exec sentora-server python scripts/prune_search_indices.py --yes
+
+Nothing here touches MySQL. The rows are still there, and the views still read
+them; this removes only the duplicate search copy.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+
+sys.path.insert(0, ".")
+
+from core import opensearch as os_utils          # noqa: E402
+
+
+def index_for(table: str) -> str:
+    return f"sentora-logs-{table.replace('_', '-')}"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--yes", action="store_true",
+                        help="delete without asking")
+    args = parser.parse_args()
+
+    client = os_utils.client
+    candidates = []
+
+    for table, reason in sorted(os_utils.NOT_INDEXED.items()):
+        index = index_for(table)
+        try:
+            if not client.indices.exists(index=index):
+                continue
+            count = client.count(index=index).get("count", 0)
+        except Exception as exc:
+            print(f"[!] {index}: could not be read ({exc}); skipping")
+            continue
+        candidates.append((index, count, reason))
+
+    if not candidates:
+        print("Nothing to prune: no index exists for a table that is no "
+              "longer written.")
+        return 0
+
+    total = sum(count for _, count, _ in candidates)
+    print(f"{len(candidates)} index/indices, {total:,} documents:\n")
+    for index, count, reason in candidates:
+        print(f"  {index:<40} {count:>10,}  {reason}")
+
+    print("\nThe rows stay in MySQL. This removes the search copy only.")
+
+    if not args.yes:
+        # input(), deliberately. A destructive default is how a maintenance
+        # script becomes the thing that lost the data.
+        answer = input("\nDelete these indices? [y/N] ").strip().lower()
+        if answer not in ("y", "yes"):
+            print("Nothing deleted.")
+            return 1
+
+    failed = 0
+    for index, count, _ in candidates:
+        try:
+            client.indices.delete(index=index)
+            print(f"[+] deleted {index} ({count:,} documents)")
+        except Exception as exc:
+            failed += 1
+            print(f"[!] {index}: {exc}")
+
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())