From 57f4c40696b31abb555dc81d4160c42cc2b7e383 Mon Sep 17 00:00:00 2001 From: Lucas Sampaio Date: Tue, 12 May 2026 16:58:27 -0700 Subject: [PATCH 1/6] Add WIP docs for eBPF L7 observability New L7 observability pages for the eBPF-based collector (tigera/calico-private#11749): - configure-bpf.mdx: enable via L7ObservabilityEnabled on FelixConfiguration; comparison with the Istio Waypoint collector - configure-istio-waypoint.mdx: placeholder - datatypes-wip.mdx: redraft of the L7 schema with a Populated by column and new protocol / collector_name / tls_* fields - aggregation.mdx: aggregation key fields with an interactive playground component - sidebar entries for all four pages Co-Authored-By: Claude Opus 4.7 (1M context) --- .../components/L7AggregationSandbox/index.js | 150 ++++++++++++++++++ .../observability/elastic/l7/aggregation.mdx | 41 +++++ .../elastic/l7/configure-bpf.mdx | 78 +++++++++ .../elastic/l7/configure-istio-waypoint.mdx | 23 +++ .../elastic/l7/datatypes-wip.mdx | 57 +++++++ sidebars-calico-enterprise.js | 9 +- 6 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 calico-enterprise/_includes/components/L7AggregationSandbox/index.js create mode 100644 calico-enterprise/observability/elastic/l7/aggregation.mdx create mode 100644 calico-enterprise/observability/elastic/l7/configure-bpf.mdx create mode 100644 calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx create mode 100644 calico-enterprise/observability/elastic/l7/datatypes-wip.mdx diff --git a/calico-enterprise/_includes/components/L7AggregationSandbox/index.js b/calico-enterprise/_includes/components/L7AggregationSandbox/index.js new file mode 100644 index 0000000000..18379a124e --- /dev/null +++ b/calico-enterprise/_includes/components/L7AggregationSandbox/index.js @@ -0,0 +1,150 @@ +import React, { useState, useMemo } from 'react'; + +const SAMPLE = [ + { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'curl/8.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart', svc: 'cart', method: 'PUT', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/products', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '404', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'checkout', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart-canary', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart', svc: 'cart-v2', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, + { src: 'frontend', dest: 'cart', svc: 'cart', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'cart.default.svc' }, + { src: 'frontend', dest: 'cart', svc: 'cart', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'payment.default.svc' }, +]; + +const AGGREGATORS = [ + { id: 'source', label: 'Source info', fields: ['src'], field: 'L7LogsFileAggregationSourceInfo' }, + { id: 'dest', label: 'Destination info', fields: ['dest'], field: 'L7LogsFileAggregationDestinationInfo' }, + { id: 'service', label: 'Service info', fields: ['svc'], field: 'L7LogsFileAggregationServiceInfo' }, + { id: 'method', label: 'HTTP method', fields: ['method'], field: 'L7LogsFileAggregationHTTPMethod' }, + { id: 'url', label: 'URL', fields: ['url'], field: 'L7LogsFileAggregationTrimURL' }, + { id: 'code', label: 'Response code', fields: ['code'], field: 'L7LogsFileAggregationResponseCode' }, + { id: 'header', label: 'HTTP header info', fields: ['ua'], field: 'L7LogsFileAggregationHTTPHeaderInfo' }, + { id: 'tlssni', label: 'TLS SNI', fields: ['sni'], field: 'L7LogsFileAggregationTLSSNI' }, +]; + +const COLUMNS = [ + { key: 'protocol', label: 'protocol', owner: null }, + { key: 'src', label: 'src_name_aggr', owner: 'source' }, + { key: 'dest', label: 'dest_name_aggr', owner: 'dest' }, + { key: 'svc', label: 'dest_service_name', owner: 'service' }, + { key: 'method', label: 'method', owner: 'method' }, + { key: 'url', label: 'url', owner: 'url' }, + { key: 'code', label: 'response_code', owner: 'code' }, + { key: 'ua', label: 'user_agent', owner: 'header' }, + { key: 'sni', label: 'tls_sni', owner: 'tlssni' }, +]; + +export default function L7AggregationSandbox() { + const [checked, setChecked] = useState( + Object.fromEntries(AGGREGATORS.map((a) => [a.id, true])), + ); + + const aggregated = useMemo(() => { + const buckets = new Map(); + for (const row of SAMPLE) { + const keyParts = [`protocol:${row.protocol}`]; + for (const a of AGGREGATORS) { + if (!checked[a.id]) continue; + for (const f of a.fields) { + keyParts.push(`${f}:${row[f] ?? ''}`); + } + } + const key = keyParts.join('|'); + if (buckets.has(key)) { + buckets.get(key).count += 1; + } else { + buckets.set(key, { ...row, count: 1 }); + } + } + return Array.from(buckets.values()); + }, [checked]); + + const toggle = (id) => + setChecked((c) => ({ ...c, [id]: !c[id] })); + + const cellValue = (row, col) => { + if (col.owner && !checked[col.owner]) return '—'; + return row[col.key] ?? '—'; + }; + + const cellStyle = { + padding: '4px 8px', + fontFamily: 'var(--ifm-font-family-monospace)', + fontSize: '0.85em', + overflowWrap: 'anywhere', + verticalAlign: 'top', + }; + const headerStyle = { + ...cellStyle, + whiteSpace: 'normal', + wordBreak: 'break-word', + }; + + return ( +
+
+ {AGGREGATORS.map((a) => ( + + ))} +
+
+ + + {COLUMNS.map((c) => ( + + ))} + + + + + {COLUMNS.map((c) => ( + + ))} + + + + + {aggregated.map((row, i) => ( + + {COLUMNS.map((c) => ( + + ))} + + + ))} + +
{c.label}count
{cellValue(row, c)}{row.count}
+
+

+ Uncheck a field to drop it from the aggregation key. Rows that differ only in dropped fields merge, and the count column increments accordingly. +

+
+ ); +} diff --git a/calico-enterprise/observability/elastic/l7/aggregation.mdx b/calico-enterprise/observability/elastic/l7/aggregation.mdx new file mode 100644 index 0000000000..431306c050 --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/aggregation.mdx @@ -0,0 +1,41 @@ +--- +description: Control how Calico Enterprise aggregates L7 log entries by selecting which fields participate in the aggregation key. +--- + +import L7AggregationSandbox from '../../../_includes/components/L7AggregationSandbox'; + +# L7 log aggregation + +## Big picture + +$[prodname] aggregates L7 events before writing them to disk. Each row in the resulting log represents a unique combination of metadata fields and carries a `count` of how many requests matched that combination during the aggregation interval. Which fields participate in that combination — the aggregation key — is controlled by a set of fields on `FelixConfiguration`. + +Including a field makes the logs more granular (more rows, fewer requests per row). Excluding a field collapses requests that differ only in that field into a single row and increments its `count` for each merged request. + +## Aggregation fields + +The following `FelixConfiguration` fields control the L7 log aggregation key. For accepted values and defaults, follow the link to the [Felix Configuration reference](../../../reference/component-resources/node/felix/configuration.mdx#l7-logs). + +| Field | Effect when included | Effect when excluded | +| --- | --- | --- | +| [L7LogsFileAggregationSourceInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationSourceInfo) | Source aggregated name, namespace, and type appear on each row. | Requests are merged across sources. | +| [L7LogsFileAggregationDestinationInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationDestinationInfo) | Destination aggregated name, namespace, and type appear on each row. | Requests are merged across destinations. | +| [L7LogsFileAggregationServiceInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationServiceInfo) | Destination service name, namespace, and port appear on each row. | Requests are merged across services. | +| [L7LogsFileAggregationHTTPMethod](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationHTTPMethod) | HTTP method appears on each row. | Requests are merged across methods. | +| [L7LogsFileAggregationTrimURL](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationTrimURL) | URL (or a trimmed prefix) appears on each row. Intermediate values trim the query string, then the path, before excluding the URL entirely. | Requests are merged across URLs. | +| [L7LogsFileAggregationResponseCode](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationResponseCode) | Response code appears on each row. | Requests are merged across response codes. | +| [L7LogsFileAggregationHTTPHeaderInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationHTTPHeaderInfo) | User agent and request type appear on each row. | Requests are merged across user agents and types. | +| [L7LogsFileAggregationTLSSNI](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationTLSSNI) | TLS Server Name Indication appears on each TLS row. | TLS flows to the same destination are merged regardless of SNI. | + +Two additional fields control how the URL is truncated before it enters the aggregation key, rather than whether the URL participates at all: + +| Field | Description | +| --- | --- | +| [L7LogsFileAggregationNumURLPath](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationNumURLPath) | Maximum number of URL path components retained. Default `5`. A negative value disables this truncation. | +| [L7LogsFileAggregationURLCharLimit](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationURLCharLimit) | Maximum URL length in characters; longer URLs are sliced. Default `250`. | + +## Aggregation playground + +The table below shows a small set of sample L7 events — eight HTTP requests and two TLS handshakes. Toggle the checkboxes to include or exclude each field from the aggregation key. The table will collapse and the `count` column will update to show how many original requests fall into each aggregated row. + + diff --git a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx new file mode 100644 index 0000000000..c8ba965228 --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx @@ -0,0 +1,78 @@ +--- +description: Capture HTTP and TLS L7 logs from kernel TCP probes using eBPF, with no proxy or sidecar injection required. +--- + +# Configure L7 logs with eBPF + +## Big picture + +Use $[prodname] eBPF-based L7 observability to capture HTTP request/response and TLS handshake metadata directly from the kernel TCP layer, without deploying a proxy or injecting sidecars. + +## Value + +L7 logs give platform operators and development teams visibility into how applications talk to each other beyond what L3/4 flow logs show: HTTP method, URL, response code, and TLS SNI / version / cipher suite. They are also key for spotting anomalous behavior such as attempts to access restricted URLs or scans for particular paths. + +The eBPF-based collector delivers this visibility with no data-path proxy and no per-deployment opt-in: enable it once at the cluster level, and HTTP and TLS metadata is captured for every workload on the node. The feature is **dataplane-independent** and works with the eBPF, iptables, and nftables dataplanes alike. + +## Concepts + +### About eBPF L7 logs + +$[prodname] attaches eBPF probes to the kernel's TCP layer and extracts HTTP request/response and TLS handshake metadata from the traffic as it flows through the kernel. The parsed metadata is forwarded to the L7 log pipeline and written to disk on each node. + +Each L7 log entry carries a `protocol` field (`http` or `tls`) and a `collector_name` field (`ebpf-tcp`) so events can be distinguished by their source. TLS entries additionally include `tls_sni`, `tls_version`, and `tls_cipher_suite`. For the full schema, see [L7 log data types](datatypes-wip.mdx). + +### Comparison with the Istio Waypoint collector + +The [Istio Waypoint collector](configure-istio-waypoint.mdx) routes selected traffic through a per-namespace Envoy waypoint proxy and produces L7 logs from the proxy itself. The eBPF collector runs inside Felix on every node and captures HTTP and TLS metadata as traffic passes through the kernel TCP layer. The two collectors can coexist in the same cluster; both feed the same L7 log pipeline and entries are distinguished by the `collector_name` field. + +| Aspect | Istio Waypoint collector | eBPF L7 collector | +| --- | --- | --- | +| Additional infrastructure | Waypoint Envoy pods per namespace/service | None — runs inside Felix | +| L7 coverage | Only traffic routed through a waypoint | All TCP traffic on the node | +| Network path overhead | Two extra proxy hops | None | +| Plaintext access | Decrypted at the waypoint (mTLS terminated) | Captured at the kernel TCP layer, before TLS encryption | +| Protocol coverage | HTTP/1, HTTP/2, gRPC | HTTP/1.0, HTTP/1.1, TLS handshake metadata | +| Opt-in granularity | Per namespace/service | Cluster-wide via FelixConfiguration | +| L7 features beyond observability | Retries, traffic splitting, fault injection | Observability only | + +## Before you begin + +### Limitations + +* Requires Linux kernel 5.17 or later on every node. +* HTTP parsing covers HTTP/1.0 and HTTP/1.1 only, because their requests and responses travel as plaintext on the wire. HTTP/2 and HTTP/3 are not captured as HTTP, and encrypted HTTP traffic surfaces only as TLS metadata. +* HTTP captures only the most recent request per socket; rapidly pipelined requests on a single connection may be undercounted. +* TLS capture is limited to handshake metadata (SNI, version, cipher suite). The collector does not decrypt application traffic. + +:::note + +L7 logs require a minimum of 1 additional GB of log storage per node, per one-day retention period. Adjust your [Log Storage](../../../operations/logstorage/adjust-log-storage-size.mdx) before you start tasks in the next section. + +::: + +## Configure L7 logs + +Enable the eBPF L7 collector by setting the [L7ObservabilityEnabled](../../../reference/component-resources/node/felix/configuration.mdx#L7ObservabilityEnabled) field to `true` on the default `FelixConfiguration`. The setting is independent of `bpfEnabled` — the collector works with any dataplane. + +```bash +kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":true}}' +``` + +To disable the collector, set the field back to `false`. Felix tears down its BPF programs and cleans up pinned maps; no node reboot is required. + +```bash +kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":false}}' +``` + +## View L7 logs + +The eBPF collector writes L7 events as JSON to a log file on each node. By default the file is at `/var/log/calico/l7logs/l7.log`. The directory is controlled by the [L7LogsFileDirectory](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileDirectory) field on `FelixConfiguration`. + +To tail the log on a node: + +```bash +tail -f /var/log/calico/l7logs/l7.log +``` + +For the JSON schema, see [L7 log data types](datatypes-wip.mdx). diff --git a/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx b/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx new file mode 100644 index 0000000000..69a13ef22e --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx @@ -0,0 +1,23 @@ +--- +description: Capture L7 logs through an Istio Ambient Mode waypoint proxy. +--- + +# Configure L7 logs with Istio Waypoint + +:::note + +This page is a placeholder. Content for L7 log collection via Istio Ambient Mode waypoint proxies is pending. + +::: + +## Big picture + +## Value + +## Concepts + +## Before you begin + +## Configure L7 logs + +## View L7 logs in the web console diff --git a/calico-enterprise/observability/elastic/l7/datatypes-wip.mdx b/calico-enterprise/observability/elastic/l7/datatypes-wip.mdx new file mode 100644 index 0000000000..e47fca38db --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/datatypes-wip.mdx @@ -0,0 +1,57 @@ +--- +description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, with the collector that populates each field. +--- + +# L7 log data types (WIP) + +:::note + +This page is a work-in-progress redraft of [L7 log data types](datatypes.mdx). It adds the fields produced by the eBPF L7 collector and a **Populated by** column that identifies which collector fills each field. + +::: + +## Big picture + +$[prodname] supports multiple L7 collectors that write into the same Elasticsearch index (`tigera_secure_ee_l7`): + +- **Envoy** — the legacy sidecar/proxy-based collector. See [Configure L7 logs](configure.mdx). +- **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Configure L7 logs with eBPF](configure-bpf.mdx). +- **Istio Waypoint** — collection through an Istio Ambient Mode waypoint proxy. See [Configure L7 logs with Istio Waypoint](configure-istio-waypoint.mdx). + +The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. A qualifier like "eBPF (TLS only)" means the field is populated only for events of that protocol family. + +:::note + +The `protocol` field (new with the eBPF collector) and the legacy `type` field both describe the wire protocol but use different value vocabularies. `protocol` is `http` or `tls`. `type` is `tcp`, `tls`, or `html/`. Filter on `protocol` for events from the eBPF collector and on `type` for events from Envoy. + +::: + +| Name | Datatype | Populated by | Description | +| ------------------------ | -------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `host` | keyword | All | Name of the node that collected the L7 log entry. | +| `start_time` | date | All | Start time of log collection in Unix timestamp format. | +| `end_time` | date | All | End time of log collection in Unix timestamp format. | +| `bytes_in` | long | All | Number of incoming bytes since the last export. | +| `bytes_out` | long | All | Number of outgoing bytes since the last export. | +| `duration_mean` | long | All | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `duration_max` | long | All | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `count` | long | All | Number of requests that match this combination of L7 data. | +| `src_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `src_namespace` | keyword | All | Namespace of the source endpoint. | +| `src_type` | keyword | All | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `dest_namespace` | keyword | All | Namespace of the destination endpoint. | +| `dest_type` | keyword | All | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_service_name` | keyword | All | Name of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_namespace` | keyword | All | Namespace of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_port` | long | All | Destination service port. | +| `url` | keyword | Envoy, eBPF (HTTP only) | URL that the request was made against. | +| `response_code` | keyword | Envoy, eBPF (HTTP only) | Response code returned by the request. | +| `method` | keyword | Envoy, eBPF (HTTP only) | HTTP method for the request. | +| `user_agent` | keyword | Envoy | User agent of the request. | +| `type` | keyword | Envoy | Type of request made. Possible values include `tcp`, `tls`, and `html/`. See the note above on `protocol` vs `type`. | +| `protocol` | keyword | eBPF | Wire-protocol family of the captured event. Possible values: `http`, `tls`. | +| `collector_name` | keyword | eBPF | Name of the collector that produced the event. For the kernel TCP-layer eBPF collector, the value is `ebpf-tcp`. | +| `tls_sni` | keyword | eBPF (TLS only) | TLS Server Name Indication sent by the client in the ClientHello. | +| `tls_version` | keyword | eBPF (TLS only) | Negotiated TLS protocol version (for example, `TLS 1.2`, `TLS 1.3`). | +| `tls_cipher_suite` | keyword | eBPF (TLS only) | Negotiated TLS cipher suite (for example, `TLS_AES_256_GCM_SHA384`). | diff --git a/sidebars-calico-enterprise.js b/sidebars-calico-enterprise.js index 5c9e7a3f73..84fffc49fd 100644 --- a/sidebars-calico-enterprise.js +++ b/sidebars-calico-enterprise.js @@ -402,7 +402,14 @@ module.exports = { type: 'category', label: 'L7 logs', link: { type: 'doc', id: 'observability/elastic/l7/index' }, - items: ['observability/elastic/l7/configure', 'observability/elastic/l7/datatypes'], + items: [ + 'observability/elastic/l7/configure', + 'observability/elastic/l7/configure-bpf', + 'observability/elastic/l7/configure-istio-waypoint', + 'observability/elastic/l7/aggregation', + 'observability/elastic/l7/datatypes', + 'observability/elastic/l7/datatypes-wip', + ], }, 'observability/elastic/troubleshoot', ], From dfbad7b14a16093f165c1dada7375b0b9f2a23eb Mon Sep 17 00:00:00 2001 From: Lucas Sampaio Date: Wed, 13 May 2026 12:27:27 -0700 Subject: [PATCH 2/6] Iterate on L7 aggregation sandbox and rename data types pages Aggregation sandbox redesign: - Group rows by aggregation bucket; expand/collapse to reveal merged events - Two-tone highlight (green for added/count up, red for removed/count down) - Color legend above the table; intro/footer condensed - More diverse sample data spanning cart, payments, and TLS scenarios Pages: - Promote the WIP data types page to datatypes.mdx; move the original to datatypes-legacy.mdx - Add collector_type and protocol_version fields - Sidebar reordered with "+ " prefix on the new pages and shorter labels (Configure with eBPF, Configure with Istio Waypoint, Aggregation, Data types) Co-Authored-By: Claude Opus 4.7 (1M context) --- .../components/L7AggregationSandbox/index.js | 267 +++++++++++++++--- .../observability/elastic/l7/aggregation.mdx | 14 +- .../elastic/l7/configure-bpf.mdx | 4 +- .../elastic/l7/datatypes-legacy.mdx | 36 +++ .../elastic/l7/datatypes-wip.mdx | 57 ---- .../observability/elastic/l7/datatypes.mdx | 65 +++-- sidebars-calico-enterprise.js | 10 +- 7 files changed, 311 insertions(+), 142 deletions(-) create mode 100644 calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx delete mode 100644 calico-enterprise/observability/elastic/l7/datatypes-wip.mdx diff --git a/calico-enterprise/_includes/components/L7AggregationSandbox/index.js b/calico-enterprise/_includes/components/L7AggregationSandbox/index.js index 18379a124e..713ed3396f 100644 --- a/calico-enterprise/_includes/components/L7AggregationSandbox/index.js +++ b/calico-enterprise/_includes/components/L7AggregationSandbox/index.js @@ -1,16 +1,16 @@ -import React, { useState, useMemo } from 'react'; +import React, { useState, useMemo, useEffect, useRef } from 'react'; const SAMPLE = [ - { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'curl/8.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart', svc: 'cart', method: 'PUT', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/products', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '404', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'checkout', dest: 'cart', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart-canary', svc: 'cart', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart', svc: 'cart-v2', method: 'GET', url: '/cart', code: '200', ua: 'cart-client/1.0', protocol: 'http', sni: null }, - { src: 'frontend', dest: 'cart', svc: 'cart', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'cart.default.svc' }, - { src: 'frontend', dest: 'cart', svc: 'cart', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'payment.default.svc' }, + { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'GET', url: '/cart', code: '200', ua: 'client', protocol: 'http', sni: null }, + { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'GET', url: '/cart', code: '404', ua: 'client', protocol: 'http', sni: null }, + { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'GET', url: '/items', code: '200', ua: 'client', protocol: 'http', sni: null }, + { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'POST', url: '/cart', code: '200', ua: 'client', protocol: 'http', sni: null }, + { src: 'orders', dest: 'pay', svc: 'pay-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk', protocol: 'http', sni: null }, + { src: 'edge', dest: 'pay', svc: 'pay-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk', protocol: 'http', sni: null }, + { src: 'orders', dest: 'canary', svc: 'pay-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk', protocol: 'http', sni: null }, + { src: 'orders', dest: 'pay', svc: 'pay-v2-svc', method: 'POST', url: '/charge', code: '200', ua: 'sdk', protocol: 'http', sni: null }, + { src: 'prom', dest: 'kibana', svc: 'kibana-svc', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'kibana.svc' }, + { src: 'prom', dest: 'kibana', svc: 'kibana-svc', method: null, url: null, code: null, ua: null, protocol: 'tls', sni: 'es.svc' }, ]; const AGGREGATORS = [ @@ -36,50 +36,144 @@ const COLUMNS = [ { key: 'sni', label: 'tls_sni', owner: 'tlssni' }, ]; +const HIGHLIGHT_MS = 1500; + export default function L7AggregationSandbox() { const [checked, setChecked] = useState( Object.fromEntries(AGGREGATORS.map((a) => [a.id, true])), ); - const aggregated = useMemo(() => { - const buckets = new Map(); - for (const row of SAMPLE) { - const keyParts = [`protocol:${row.protocol}`]; + const displayedRows = useMemo(() => { + const bucketKeys = SAMPLE.map((row) => { + const parts = [`protocol:${row.protocol}`]; for (const a of AGGREGATORS) { if (!checked[a.id]) continue; - for (const f of a.fields) { - keyParts.push(`${f}:${row[f] ?? ''}`); - } + for (const f of a.fields) parts.push(`${f}:${row[f] ?? ''}`); } - const key = keyParts.join('|'); - if (buckets.has(key)) { - buckets.get(key).count += 1; - } else { - buckets.set(key, { ...row, count: 1 }); + return parts.join('|'); + }); + + const bucketCounts = new Map(); + const bucketOrder = new Map(); + let nextBucketIdx = 0; + for (const k of bucketKeys) { + bucketCounts.set(k, (bucketCounts.get(k) || 0) + 1); + if (!bucketOrder.has(k)) { + bucketOrder.set(k, nextBucketIdx++); } } - return Array.from(buckets.values()); + const primaryRowForBucket = new Map(); + + const rows = SAMPLE.map((row, originalIdx) => { + const k = bucketKeys[originalIdx]; + if (!primaryRowForBucket.has(k)) primaryRowForBucket.set(k, originalIdx); + const primaryOrigIdx = primaryRowForBucket.get(k); + const isPrimary = primaryOrigIdx === originalIdx; + const bucketIdx = bucketOrder.get(k); + const bucketSize = bucketCounts.get(k); + const out = { + originalIdx, + bucketIdx, + primaryOrigIdx, + bucketSize, + count: isPrimary ? bucketSize : '', + __primary: isPrimary, + }; + for (const col of COLUMNS) { + out[col.key] = + col.owner && !checked[col.owner] ? '—' : row[col.key] ?? '—'; + } + return out; + }); + + rows.sort((a, b) => { + if (a.bucketIdx !== b.bucketIdx) return a.bucketIdx - b.bucketIdx; + return a.originalIdx - b.originalIdx; + }); + + return rows; }, [checked]); - const toggle = (id) => - setChecked((c) => ({ ...c, [id]: !c[id] })); + const prevRef = useRef(displayedRows); + const [highlights, setHighlights] = useState(() => new Map()); + const [expanded, setExpanded] = useState(() => new Set()); - const cellValue = (row, col) => { - if (col.owner && !checked[col.owner]) return '—'; - return row[col.key] ?? '—'; + const toggleExpand = (originalIdx) => { + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(originalIdx)) next.delete(originalIdx); + else next.add(originalIdx); + return next; + }); }; - const cellStyle = { + const visibleRows = displayedRows.filter( + (row) => row.__primary || expanded.has(row.primaryOrigIdx), + ); + + useEffect(() => { + const newHL = new Map(); + const prevByOrigIdx = new Map(); + for (const r of prevRef.current) { + prevByOrigIdx.set(r.originalIdx, r); + } + const HIGHLIGHTABLE = ['count', ...COLUMNS.map((c) => c.key)]; + const numOrZero = (v) => (typeof v === 'number' ? v : 0); + for (const row of displayedRows) { + const prev = prevByOrigIdx.get(row.originalIdx) || {}; + for (const k of HIGHLIGHTABLE) { + if (row[k] === prev[k]) continue; + let direction; + if (k === 'count') { + direction = numOrZero(row[k]) > numOrZero(prev[k]) ? 'add' : 'remove'; + } else { + direction = row[k] === '—' ? 'remove' : 'add'; + } + newHL.set(`${row.originalIdx}:${k}`, direction); + } + } + prevRef.current = displayedRows; + setHighlights(newHL); + if (newHL.size === 0) return; + const t = setTimeout(() => setHighlights(new Map()), HIGHLIGHT_MS); + return () => clearTimeout(t); + }, [displayedRows]); + + const toggle = (id) => setChecked((c) => ({ ...c, [id]: !c[id] })); + + const ROW_HEIGHT = '3.2em'; + const baseCellStyle = { padding: '4px 8px', fontFamily: 'var(--ifm-font-family-monospace)', fontSize: '0.85em', overflowWrap: 'anywhere', verticalAlign: 'top', + transition: 'background-color 1s ease', + height: ROW_HEIGHT, + boxSizing: 'border-box', + backgroundColor: 'transparent', + }; + const cellStyleFor = (rowIdx, key) => { + const direction = highlights.get(`${rowIdx}:${key}`); + if (direction === 'add') { + return { + ...baseCellStyle, + backgroundColor: 'var(--ifm-color-success-contrast-background)', + }; + } + if (direction === 'remove') { + return { + ...baseCellStyle, + backgroundColor: 'var(--ifm-color-danger-contrast-background)', + }; + } + return baseCellStyle; }; const headerStyle = { - ...cellStyle, + ...baseCellStyle, whiteSpace: 'normal', wordBreak: 'break-word', + transition: 'none', }; return ( @@ -114,37 +208,122 @@ export default function L7AggregationSandbox() { ))} +
+ + + added (value gained or count up) + + + + removed (value lost or count down) + +
+ {COLUMNS.map((c) => ( ))} - + {COLUMNS.map((c) => ( ))} - - {aggregated.map((row, i) => ( - - {COLUMNS.map((c) => ( - - ))} - - - ))} + {visibleRows.map((row, displayIdx) => { + const prevRow = visibleRows[displayIdx - 1]; + const isBucketStart = + !prevRow || prevRow.bucketIdx !== row.bucketIdx; + const borderTop = isBucketStart && displayIdx > 0 + ? '2px solid var(--ifm-color-emphasis-300)' + : undefined; + const styleForCell = (key) => { + const base = cellStyleFor(row.originalIdx, key); + return borderTop ? { ...base, borderTop } : base; + }; + const rowOpacity = row.__primary ? 1 : 0.7; + const isExpandable = row.__primary && row.bucketSize > 1; + const isOpen = expanded.has(row.originalIdx); + return ( + + + {COLUMNS.map((c) => ( + + ))} + + ); + })}
count{c.label}count
{cellValue(row, c)}{row.count}
+ {isExpandable ? ( + + ) : null} + {row.count} + {row[c.key]}
-

- Uncheck a field to drop it from the aggregation key. Rows that differ only in dropped fields merge, and the count column increments accordingly. -

); } diff --git a/calico-enterprise/observability/elastic/l7/aggregation.mdx b/calico-enterprise/observability/elastic/l7/aggregation.mdx index 431306c050..fcd76d6ec3 100644 --- a/calico-enterprise/observability/elastic/l7/aggregation.mdx +++ b/calico-enterprise/observability/elastic/l7/aggregation.mdx @@ -8,10 +8,16 @@ import L7AggregationSandbox from '../../../_includes/components/L7AggregationSan ## Big picture -$[prodname] aggregates L7 events before writing them to disk. Each row in the resulting log represents a unique combination of metadata fields and carries a `count` of how many requests matched that combination during the aggregation interval. Which fields participate in that combination — the aggregation key — is controlled by a set of fields on `FelixConfiguration`. +$[prodname] aggregates L7 events before writing them to disk. Each row in the resulting log represents a unique combination of metadata fields and carries a `count` of how many requests matched that combination during the aggregation interval. Which fields participate in that combination — the aggregation key — is controlled by a set of fields on [Felix Configuration](../../../reference/component-resources/node/felix/configuration.mdx#l7-logs). Including a field makes the logs more granular (more rows, fewer requests per row). Excluding a field collapses requests that differ only in that field into a single row and increments its `count` for each merged request. +## Aggregation playground + +The table below shows a small set of sample L7 events. Each visible row is what would be written to the L7 log, with `count` equal to the number of requests merged into it. Toggle the checkboxes to include or exclude each field from the aggregation key. + + + ## Aggregation fields The following `FelixConfiguration` fields control the L7 log aggregation key. For accepted values and defaults, follow the link to the [Felix Configuration reference](../../../reference/component-resources/node/felix/configuration.mdx#l7-logs). @@ -33,9 +39,3 @@ Two additional fields control how the URL is truncated before it enters the aggr | --- | --- | | [L7LogsFileAggregationNumURLPath](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationNumURLPath) | Maximum number of URL path components retained. Default `5`. A negative value disables this truncation. | | [L7LogsFileAggregationURLCharLimit](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationURLCharLimit) | Maximum URL length in characters; longer URLs are sliced. Default `250`. | - -## Aggregation playground - -The table below shows a small set of sample L7 events — eight HTTP requests and two TLS handshakes. Toggle the checkboxes to include or exclude each field from the aggregation key. The table will collapse and the `count` column will update to show how many original requests fall into each aggregated row. - - diff --git a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx index c8ba965228..f0a190a13f 100644 --- a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx +++ b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx @@ -20,7 +20,7 @@ The eBPF-based collector delivers this visibility with no data-path proxy and no $[prodname] attaches eBPF probes to the kernel's TCP layer and extracts HTTP request/response and TLS handshake metadata from the traffic as it flows through the kernel. The parsed metadata is forwarded to the L7 log pipeline and written to disk on each node. -Each L7 log entry carries a `protocol` field (`http` or `tls`) and a `collector_name` field (`ebpf-tcp`) so events can be distinguished by their source. TLS entries additionally include `tls_sni`, `tls_version`, and `tls_cipher_suite`. For the full schema, see [L7 log data types](datatypes-wip.mdx). +Each L7 log entry carries a `protocol` field (`http` or `tls`) and a `collector_name` field (`ebpf-tcp`) so events can be distinguished by their source. TLS entries additionally include `tls_sni`, `tls_version`, and `tls_cipher_suite`. For the full schema, see [L7 log data types](datatypes.mdx). ### Comparison with the Istio Waypoint collector @@ -75,4 +75,4 @@ To tail the log on a node: tail -f /var/log/calico/l7logs/l7.log ``` -For the JSON schema, see [L7 log data types](datatypes-wip.mdx). +For the JSON schema, see [L7 log data types](datatypes.mdx). diff --git a/calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx b/calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx new file mode 100644 index 0000000000..478adeeb37 --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx @@ -0,0 +1,36 @@ +--- +description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, including durations, byte counts, and HTTP request metadata. +--- + +# L7 log data types + +## Big picture + +$[prodname] sends the following data to Elasticsearch. + +The following table details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html). + +| Name | Datatype | Description | +| ------------------------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `host` | keyword | Name of the node that collected the L7 log entry. | +| `start_time` | date | Start time of log collection in Unix timestamp format. | +| `end_time` | date | End time of log collection in Unix timestamp format. | +| `bytes_in` | long | Number of incoming bytes since the last export. | +| `bytes_out` | long | Number of outgoing bytes since the last export. | +| `duration_mean` | long | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `duration_max` | long | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `count` | long | Number of requests that match this combination of L7 data. | +| `src_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `src_namespace` | keyword | Namespace of the source endpoint. | +| `src_type` | keyword | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `dest_namespace` | keyword | Namespace of the destination endpoint. | +| `dest_type` | keyword | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_service_name` | keyword | Name of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_namespace` | keyword | Namespace of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_port` | long | Destination service port. | +| `url` | keyword | URL that the request was made against. | +| `response_code` | keyword | Response code returned by the request. | +| `method` | keyword | HTTP method for the request. | +| `user_agent` | keyword | User agent of the request. | +| `type` | keyword | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | diff --git a/calico-enterprise/observability/elastic/l7/datatypes-wip.mdx b/calico-enterprise/observability/elastic/l7/datatypes-wip.mdx deleted file mode 100644 index e47fca38db..0000000000 --- a/calico-enterprise/observability/elastic/l7/datatypes-wip.mdx +++ /dev/null @@ -1,57 +0,0 @@ ---- -description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, with the collector that populates each field. ---- - -# L7 log data types (WIP) - -:::note - -This page is a work-in-progress redraft of [L7 log data types](datatypes.mdx). It adds the fields produced by the eBPF L7 collector and a **Populated by** column that identifies which collector fills each field. - -::: - -## Big picture - -$[prodname] supports multiple L7 collectors that write into the same Elasticsearch index (`tigera_secure_ee_l7`): - -- **Envoy** — the legacy sidecar/proxy-based collector. See [Configure L7 logs](configure.mdx). -- **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Configure L7 logs with eBPF](configure-bpf.mdx). -- **Istio Waypoint** — collection through an Istio Ambient Mode waypoint proxy. See [Configure L7 logs with Istio Waypoint](configure-istio-waypoint.mdx). - -The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. A qualifier like "eBPF (TLS only)" means the field is populated only for events of that protocol family. - -:::note - -The `protocol` field (new with the eBPF collector) and the legacy `type` field both describe the wire protocol but use different value vocabularies. `protocol` is `http` or `tls`. `type` is `tcp`, `tls`, or `html/`. Filter on `protocol` for events from the eBPF collector and on `type` for events from Envoy. - -::: - -| Name | Datatype | Populated by | Description | -| ------------------------ | -------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `host` | keyword | All | Name of the node that collected the L7 log entry. | -| `start_time` | date | All | Start time of log collection in Unix timestamp format. | -| `end_time` | date | All | End time of log collection in Unix timestamp format. | -| `bytes_in` | long | All | Number of incoming bytes since the last export. | -| `bytes_out` | long | All | Number of outgoing bytes since the last export. | -| `duration_mean` | long | All | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | -| `duration_max` | long | All | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | -| `count` | long | All | Number of requests that match this combination of L7 data. | -| `src_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | -| `src_namespace` | keyword | All | Namespace of the source endpoint. | -| `src_type` | keyword | All | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | -| `dest_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | -| `dest_namespace` | keyword | All | Namespace of the destination endpoint. | -| `dest_type` | keyword | All | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | -| `dest_service_name` | keyword | All | Name of the destination service. This may be empty if the request was not made against a service. | -| `dest_service_namespace` | keyword | All | Namespace of the destination service. This may be empty if the request was not made against a service. | -| `dest_service_port` | long | All | Destination service port. | -| `url` | keyword | Envoy, eBPF (HTTP only) | URL that the request was made against. | -| `response_code` | keyword | Envoy, eBPF (HTTP only) | Response code returned by the request. | -| `method` | keyword | Envoy, eBPF (HTTP only) | HTTP method for the request. | -| `user_agent` | keyword | Envoy | User agent of the request. | -| `type` | keyword | Envoy | Type of request made. Possible values include `tcp`, `tls`, and `html/`. See the note above on `protocol` vs `type`. | -| `protocol` | keyword | eBPF | Wire-protocol family of the captured event. Possible values: `http`, `tls`. | -| `collector_name` | keyword | eBPF | Name of the collector that produced the event. For the kernel TCP-layer eBPF collector, the value is `ebpf-tcp`. | -| `tls_sni` | keyword | eBPF (TLS only) | TLS Server Name Indication sent by the client in the ClientHello. | -| `tls_version` | keyword | eBPF (TLS only) | Negotiated TLS protocol version (for example, `TLS 1.2`, `TLS 1.3`). | -| `tls_cipher_suite` | keyword | eBPF (TLS only) | Negotiated TLS cipher suite (for example, `TLS_AES_256_GCM_SHA384`). | diff --git a/calico-enterprise/observability/elastic/l7/datatypes.mdx b/calico-enterprise/observability/elastic/l7/datatypes.mdx index 478adeeb37..bcc8e969b4 100644 --- a/calico-enterprise/observability/elastic/l7/datatypes.mdx +++ b/calico-enterprise/observability/elastic/l7/datatypes.mdx @@ -1,36 +1,47 @@ --- -description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, including durations, byte counts, and HTTP request metadata. +description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, with the collector that populates each field. --- # L7 log data types ## Big picture -$[prodname] sends the following data to Elasticsearch. +$[prodname] supports multiple L7 collectors that write into the same Elasticsearch index: -The following table details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html). +- **Envoy** — the legacy sidecar/proxy-based collector. See [Configure L7 logs](configure.mdx). +- **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Configure L7 logs with eBPF](configure-bpf.mdx). +- **Waypoint** — collection through an Istio Ambient Mode waypoint proxy. See [Configure L7 logs with Istio Waypoint](configure-istio-waypoint.mdx). -| Name | Datatype | Description | -| ------------------------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `host` | keyword | Name of the node that collected the L7 log entry. | -| `start_time` | date | Start time of log collection in Unix timestamp format. | -| `end_time` | date | End time of log collection in Unix timestamp format. | -| `bytes_in` | long | Number of incoming bytes since the last export. | -| `bytes_out` | long | Number of outgoing bytes since the last export. | -| `duration_mean` | long | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | -| `duration_max` | long | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | -| `count` | long | Number of requests that match this combination of L7 data. | -| `src_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | -| `src_namespace` | keyword | Namespace of the source endpoint. | -| `src_type` | keyword | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | -| `dest_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | -| `dest_namespace` | keyword | Namespace of the destination endpoint. | -| `dest_type` | keyword | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | -| `dest_service_name` | keyword | Name of the destination service. This may be empty if the request was not made against a service. | -| `dest_service_namespace` | keyword | Namespace of the destination service. This may be empty if the request was not made against a service. | -| `dest_service_port` | long | Destination service port. | -| `url` | keyword | URL that the request was made against. | -| `response_code` | keyword | Response code returned by the request. | -| `method` | keyword | HTTP method for the request. | -| `user_agent` | keyword | User agent of the request. | -| `type` | keyword | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | +The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. A qualifier like "eBPF (TLS only)" means the field is populated only for events of that protocol family. + +| Name | Datatype | Populated by | Description | +| ------------------------ | -------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `host` | keyword | All | Name of the node that collected the L7 log entry. | +| `start_time` | date | All | Start time of log collection in Unix timestamp format. | +| `end_time` | date | All | End time of log collection in Unix timestamp format. | +| `bytes_in` | long | All | Number of incoming bytes since the last export. | +| `bytes_out` | long | All | Number of outgoing bytes since the last export. | +| `duration_mean` | long | All | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `duration_max` | long | All | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `count` | long | All | Number of requests that match this combination of L7 data. | +| `src_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `src_namespace` | keyword | All | Namespace of the source endpoint. | +| `src_type` | keyword | All | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `dest_namespace` | keyword | All | Namespace of the destination endpoint. | +| `dest_type` | keyword | All | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_service_name` | keyword | All | Name of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_namespace` | keyword | All | Namespace of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_port` | long | All | Destination service port. | +| `url` | keyword | All (HTTP only) | URL that the request was made against. | +| `response_code` | keyword | All (HTTP only) | Response code returned by the request. | +| `method` | keyword | All (HTTP only) | HTTP method for the request. | +| `user_agent` | keyword | Envoy, Waypoint | User agent of the request. | +| `type` | keyword | Envoy, Waypoint | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | +| `protocol` | keyword | eBPF, Waypoint | Wire-protocol family of the captured event. Possible values: `http`, `tls`. | +| `protocol_version` | keyword | eBPF, Waypoint | Wire-protocol version. For HTTP events, possible values include `1.0` and `1.1`. | +| `collector_type` | keyword | eBPF, Waypoint | Category of the collector that produced the event. Possible values include `ebpf` and `waypoint`. | +| `collector_name` | keyword | eBPF, Waypoint | Name of the specific collector that produced the event. For the kernel TCP-layer eBPF collector, the value is `ebpf-tcp`. | +| `tls_sni` | keyword | eBPF (TLS only) | TLS Server Name Indication sent by the client in the ClientHello. | +| `tls_version` | keyword | eBPF (TLS only) | Negotiated TLS protocol version (for example, `TLS 1.2`, `TLS 1.3`). | +| `tls_cipher_suite` | keyword | eBPF (TLS only) | Negotiated TLS cipher suite (for example, `TLS_AES_256_GCM_SHA384`). | diff --git a/sidebars-calico-enterprise.js b/sidebars-calico-enterprise.js index 84fffc49fd..b5835dbc7a 100644 --- a/sidebars-calico-enterprise.js +++ b/sidebars-calico-enterprise.js @@ -403,12 +403,12 @@ module.exports = { label: 'L7 logs', link: { type: 'doc', id: 'observability/elastic/l7/index' }, items: [ + { type: 'doc', id: 'observability/elastic/l7/configure-bpf', label: '+ Configure with eBPF' }, + { type: 'doc', id: 'observability/elastic/l7/configure-istio-waypoint', label: '+ Configure with Istio Waypoint' }, + { type: 'doc', id: 'observability/elastic/l7/datatypes', label: '+ Data types' }, + { type: 'doc', id: 'observability/elastic/l7/aggregation', label: '+ Aggregation' }, 'observability/elastic/l7/configure', - 'observability/elastic/l7/configure-bpf', - 'observability/elastic/l7/configure-istio-waypoint', - 'observability/elastic/l7/aggregation', - 'observability/elastic/l7/datatypes', - 'observability/elastic/l7/datatypes-wip', + { type: 'doc', id: 'observability/elastic/l7/datatypes-legacy', label: 'L7 log data types' }, ], }, 'observability/elastic/troubleshoot', From 1eef8f32234f84b89ead5e92183b1aea9002e302 Mon Sep 17 00:00:00 2001 From: Lucas Sampaio Date: Wed, 13 May 2026 13:28:26 -0700 Subject: [PATCH 3/6] Fix Vale lint errors on new L7 pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - dataplane / dataplanes → data plane / data planes - waypoint → Waypoint (Calico term rule) - Plaintext / plaintext → Plain text / plain text - Rephrase HTTP pipelining bullet to drop dictionary-rejected words Co-Authored-By: Claude Opus 4.7 (1M context) --- .../observability/elastic/l7/configure-bpf.mdx | 14 +++++++------- .../elastic/l7/configure-istio-waypoint.mdx | 4 ++-- .../observability/elastic/l7/datatypes.mdx | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx index f0a190a13f..35febe09bf 100644 --- a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx +++ b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx @@ -12,7 +12,7 @@ Use $[prodname] eBPF-based L7 observability to capture HTTP request/response and L7 logs give platform operators and development teams visibility into how applications talk to each other beyond what L3/4 flow logs show: HTTP method, URL, response code, and TLS SNI / version / cipher suite. They are also key for spotting anomalous behavior such as attempts to access restricted URLs or scans for particular paths. -The eBPF-based collector delivers this visibility with no data-path proxy and no per-deployment opt-in: enable it once at the cluster level, and HTTP and TLS metadata is captured for every workload on the node. The feature is **dataplane-independent** and works with the eBPF, iptables, and nftables dataplanes alike. +The eBPF-based collector delivers this visibility with no data-path proxy and no per-deployment opt-in: enable it once at the cluster level, and HTTP and TLS metadata is captured for every workload on the node. The feature is **data-plane-independent** and works with the eBPF, iptables, and nftables data planes alike. ## Concepts @@ -24,14 +24,14 @@ Each L7 log entry carries a `protocol` field (`http` or `tls`) and a `collector_ ### Comparison with the Istio Waypoint collector -The [Istio Waypoint collector](configure-istio-waypoint.mdx) routes selected traffic through a per-namespace Envoy waypoint proxy and produces L7 logs from the proxy itself. The eBPF collector runs inside Felix on every node and captures HTTP and TLS metadata as traffic passes through the kernel TCP layer. The two collectors can coexist in the same cluster; both feed the same L7 log pipeline and entries are distinguished by the `collector_name` field. +The [Istio Waypoint collector](configure-istio-waypoint.mdx) routes selected traffic through a per-namespace Envoy Waypoint proxy and produces L7 logs from the proxy itself. The eBPF collector runs inside Felix on every node and captures HTTP and TLS metadata as traffic passes through the kernel TCP layer. The two collectors can coexist in the same cluster; both feed the same L7 log pipeline and entries are distinguished by the `collector_name` field. | Aspect | Istio Waypoint collector | eBPF L7 collector | | --- | --- | --- | | Additional infrastructure | Waypoint Envoy pods per namespace/service | None — runs inside Felix | -| L7 coverage | Only traffic routed through a waypoint | All TCP traffic on the node | +| L7 coverage | Only traffic routed through a Waypoint | All TCP traffic on the node | | Network path overhead | Two extra proxy hops | None | -| Plaintext access | Decrypted at the waypoint (mTLS terminated) | Captured at the kernel TCP layer, before TLS encryption | +| Plain text access | Decrypted at the Waypoint (mTLS terminated) | Captured at the kernel TCP layer, before TLS encryption | | Protocol coverage | HTTP/1, HTTP/2, gRPC | HTTP/1.0, HTTP/1.1, TLS handshake metadata | | Opt-in granularity | Per namespace/service | Cluster-wide via FelixConfiguration | | L7 features beyond observability | Retries, traffic splitting, fault injection | Observability only | @@ -41,8 +41,8 @@ The [Istio Waypoint collector](configure-istio-waypoint.mdx) routes selected tra ### Limitations * Requires Linux kernel 5.17 or later on every node. -* HTTP parsing covers HTTP/1.0 and HTTP/1.1 only, because their requests and responses travel as plaintext on the wire. HTTP/2 and HTTP/3 are not captured as HTTP, and encrypted HTTP traffic surfaces only as TLS metadata. -* HTTP captures only the most recent request per socket; rapidly pipelined requests on a single connection may be undercounted. +* HTTP parsing covers HTTP/1.0 and HTTP/1.1 only, because their requests and responses travel as plain text on the wire. HTTP/2 and HTTP/3 are not captured as HTTP, and encrypted HTTP traffic surfaces only as TLS metadata. +* HTTP capture tracks only the most recent request per socket; when a single connection carries multiple back-to-back requests, some may be missed. * TLS capture is limited to handshake metadata (SNI, version, cipher suite). The collector does not decrypt application traffic. :::note @@ -53,7 +53,7 @@ L7 logs require a minimum of 1 additional GB of log storage per node, per one-da ## Configure L7 logs -Enable the eBPF L7 collector by setting the [L7ObservabilityEnabled](../../../reference/component-resources/node/felix/configuration.mdx#L7ObservabilityEnabled) field to `true` on the default `FelixConfiguration`. The setting is independent of `bpfEnabled` — the collector works with any dataplane. +Enable the eBPF L7 collector by setting the [L7ObservabilityEnabled](../../../reference/component-resources/node/felix/configuration.mdx#L7ObservabilityEnabled) field to `true` on the default `FelixConfiguration`. The setting is independent of `bpfEnabled` — the collector works with any data plane. ```bash kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":true}}' diff --git a/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx b/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx index 69a13ef22e..173fe8a852 100644 --- a/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx +++ b/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx @@ -1,12 +1,12 @@ --- -description: Capture L7 logs through an Istio Ambient Mode waypoint proxy. +description: Capture L7 logs through an Istio Ambient Mode Waypoint proxy. --- # Configure L7 logs with Istio Waypoint :::note -This page is a placeholder. Content for L7 log collection via Istio Ambient Mode waypoint proxies is pending. +This page is a placeholder. Content for L7 log collection via Istio Ambient Mode Waypoint proxies is pending. ::: diff --git a/calico-enterprise/observability/elastic/l7/datatypes.mdx b/calico-enterprise/observability/elastic/l7/datatypes.mdx index bcc8e969b4..1e98127fa1 100644 --- a/calico-enterprise/observability/elastic/l7/datatypes.mdx +++ b/calico-enterprise/observability/elastic/l7/datatypes.mdx @@ -10,7 +10,7 @@ $[prodname] supports multiple L7 collectors that write into the same Elasticsear - **Envoy** — the legacy sidecar/proxy-based collector. See [Configure L7 logs](configure.mdx). - **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Configure L7 logs with eBPF](configure-bpf.mdx). -- **Waypoint** — collection through an Istio Ambient Mode waypoint proxy. See [Configure L7 logs with Istio Waypoint](configure-istio-waypoint.mdx). +- **Waypoint** — collection through an Istio Ambient Mode Waypoint proxy. See [Configure L7 logs with Istio Waypoint](configure-istio-waypoint.mdx). The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. A qualifier like "eBPF (TLS only)" means the field is populated only for events of that protocol family. From 519d74b4219a7b6577ffb6a854becfe8433d6a98 Mon Sep 17 00:00:00 2001 From: Lucas Sampaio Date: Fri, 22 May 2026 10:34:56 -0700 Subject: [PATCH 4/6] Restructure L7 logs docs: add Overview, rename collector pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add Overview page (overview.mdx) with big picture, value, collectors, and a comparison table — moves the shared framing out of per-collector pages - Rename configure-bpf.mdx → enable-ebpf.mdx, refocus on enablement steps - Rename configure-istio-waypoint.mdx → enable-waypoint.mdx (placeholder) - Drop datatypes-legacy.mdx (no inbound references) - Sidebar labels: Overview, Enable eBPF/Waypoint/Envoy collector, Data types, Aggregation - Datatypes page: HTTP-only / TLS-only fields tagged inline in the description - Repoint external "L7 logs" links (dashboards, get-started-cem, kibana) to overview.mdx Co-Authored-By: Claude Opus 4.7 (1M context) --- .../observability/dashboards.mdx | 2 +- .../observability/elastic/l7/aggregation.mdx | 20 +---- .../elastic/l7/configure-bpf.mdx | 78 ------------------- .../elastic/l7/configure-istio-waypoint.mdx | 23 ------ .../elastic/l7/datatypes-legacy.mdx | 36 --------- .../observability/elastic/l7/datatypes.mdx | 26 +++---- .../observability/elastic/l7/enable-ebpf.mdx | 42 ++++++++++ .../elastic/l7/enable-waypoint.mdx | 19 +++++ .../observability/elastic/l7/overview.mdx | 53 +++++++++++++ .../observability/get-started-cem.mdx | 2 +- calico-enterprise/observability/kibana.mdx | 2 +- sidebars-calico-enterprise.js | 12 +-- 12 files changed, 137 insertions(+), 178 deletions(-) delete mode 100644 calico-enterprise/observability/elastic/l7/configure-bpf.mdx delete mode 100644 calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx delete mode 100644 calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx create mode 100644 calico-enterprise/observability/elastic/l7/enable-ebpf.mdx create mode 100644 calico-enterprise/observability/elastic/l7/enable-waypoint.mdx create mode 100644 calico-enterprise/observability/elastic/l7/overview.mdx diff --git a/calico-enterprise/observability/dashboards.mdx b/calico-enterprise/observability/dashboards.mdx index e2ef910e64..363ea1f048 100644 --- a/calico-enterprise/observability/dashboards.mdx +++ b/calico-enterprise/observability/dashboards.mdx @@ -49,7 +49,7 @@ Seeing this data helps you spot unusual flow activity, which may indicate a comp The **HTTP Traffic** dashboard provides application performance metrics for in-scope Kubernetes services. The data can assist service owners and platform personnel in assessing the health of cluster workloads without the need for a full service mesh. -[L7 logs](elastic/l7/configure.mdx) are not enabled by default, and must be configured. +[L7 logs](elastic/l7/overview.mdx) are not enabled by default, and must be configured. diff --git a/calico-enterprise/observability/elastic/l7/aggregation.mdx b/calico-enterprise/observability/elastic/l7/aggregation.mdx index fcd76d6ec3..d5062f3c16 100644 --- a/calico-enterprise/observability/elastic/l7/aggregation.mdx +++ b/calico-enterprise/observability/elastic/l7/aggregation.mdx @@ -20,22 +20,4 @@ The table below shows a small set of sample L7 events. Each visible row is what ## Aggregation fields -The following `FelixConfiguration` fields control the L7 log aggregation key. For accepted values and defaults, follow the link to the [Felix Configuration reference](../../../reference/component-resources/node/felix/configuration.mdx#l7-logs). - -| Field | Effect when included | Effect when excluded | -| --- | --- | --- | -| [L7LogsFileAggregationSourceInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationSourceInfo) | Source aggregated name, namespace, and type appear on each row. | Requests are merged across sources. | -| [L7LogsFileAggregationDestinationInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationDestinationInfo) | Destination aggregated name, namespace, and type appear on each row. | Requests are merged across destinations. | -| [L7LogsFileAggregationServiceInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationServiceInfo) | Destination service name, namespace, and port appear on each row. | Requests are merged across services. | -| [L7LogsFileAggregationHTTPMethod](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationHTTPMethod) | HTTP method appears on each row. | Requests are merged across methods. | -| [L7LogsFileAggregationTrimURL](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationTrimURL) | URL (or a trimmed prefix) appears on each row. Intermediate values trim the query string, then the path, before excluding the URL entirely. | Requests are merged across URLs. | -| [L7LogsFileAggregationResponseCode](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationResponseCode) | Response code appears on each row. | Requests are merged across response codes. | -| [L7LogsFileAggregationHTTPHeaderInfo](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationHTTPHeaderInfo) | User agent and request type appear on each row. | Requests are merged across user agents and types. | -| [L7LogsFileAggregationTLSSNI](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationTLSSNI) | TLS Server Name Indication appears on each TLS row. | TLS flows to the same destination are merged regardless of SNI. | - -Two additional fields control how the URL is truncated before it enters the aggregation key, rather than whether the URL participates at all: - -| Field | Description | -| --- | --- | -| [L7LogsFileAggregationNumURLPath](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationNumURLPath) | Maximum number of URL path components retained. Default `5`. A negative value disables this truncation. | -| [L7LogsFileAggregationURLCharLimit](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileAggregationURLCharLimit) | Maximum URL length in characters; longer URLs are sliced. Default `250`. | +The `FelixConfiguration` fields that control the L7 log aggregation key, including their accepted values and default settings, are documented in the [Felix Configuration reference](../../../reference/component-resources/node/felix/configuration.mdx#l7-logs). diff --git a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx b/calico-enterprise/observability/elastic/l7/configure-bpf.mdx deleted file mode 100644 index 35febe09bf..0000000000 --- a/calico-enterprise/observability/elastic/l7/configure-bpf.mdx +++ /dev/null @@ -1,78 +0,0 @@ ---- -description: Capture HTTP and TLS L7 logs from kernel TCP probes using eBPF, with no proxy or sidecar injection required. ---- - -# Configure L7 logs with eBPF - -## Big picture - -Use $[prodname] eBPF-based L7 observability to capture HTTP request/response and TLS handshake metadata directly from the kernel TCP layer, without deploying a proxy or injecting sidecars. - -## Value - -L7 logs give platform operators and development teams visibility into how applications talk to each other beyond what L3/4 flow logs show: HTTP method, URL, response code, and TLS SNI / version / cipher suite. They are also key for spotting anomalous behavior such as attempts to access restricted URLs or scans for particular paths. - -The eBPF-based collector delivers this visibility with no data-path proxy and no per-deployment opt-in: enable it once at the cluster level, and HTTP and TLS metadata is captured for every workload on the node. The feature is **data-plane-independent** and works with the eBPF, iptables, and nftables data planes alike. - -## Concepts - -### About eBPF L7 logs - -$[prodname] attaches eBPF probes to the kernel's TCP layer and extracts HTTP request/response and TLS handshake metadata from the traffic as it flows through the kernel. The parsed metadata is forwarded to the L7 log pipeline and written to disk on each node. - -Each L7 log entry carries a `protocol` field (`http` or `tls`) and a `collector_name` field (`ebpf-tcp`) so events can be distinguished by their source. TLS entries additionally include `tls_sni`, `tls_version`, and `tls_cipher_suite`. For the full schema, see [L7 log data types](datatypes.mdx). - -### Comparison with the Istio Waypoint collector - -The [Istio Waypoint collector](configure-istio-waypoint.mdx) routes selected traffic through a per-namespace Envoy Waypoint proxy and produces L7 logs from the proxy itself. The eBPF collector runs inside Felix on every node and captures HTTP and TLS metadata as traffic passes through the kernel TCP layer. The two collectors can coexist in the same cluster; both feed the same L7 log pipeline and entries are distinguished by the `collector_name` field. - -| Aspect | Istio Waypoint collector | eBPF L7 collector | -| --- | --- | --- | -| Additional infrastructure | Waypoint Envoy pods per namespace/service | None — runs inside Felix | -| L7 coverage | Only traffic routed through a Waypoint | All TCP traffic on the node | -| Network path overhead | Two extra proxy hops | None | -| Plain text access | Decrypted at the Waypoint (mTLS terminated) | Captured at the kernel TCP layer, before TLS encryption | -| Protocol coverage | HTTP/1, HTTP/2, gRPC | HTTP/1.0, HTTP/1.1, TLS handshake metadata | -| Opt-in granularity | Per namespace/service | Cluster-wide via FelixConfiguration | -| L7 features beyond observability | Retries, traffic splitting, fault injection | Observability only | - -## Before you begin - -### Limitations - -* Requires Linux kernel 5.17 or later on every node. -* HTTP parsing covers HTTP/1.0 and HTTP/1.1 only, because their requests and responses travel as plain text on the wire. HTTP/2 and HTTP/3 are not captured as HTTP, and encrypted HTTP traffic surfaces only as TLS metadata. -* HTTP capture tracks only the most recent request per socket; when a single connection carries multiple back-to-back requests, some may be missed. -* TLS capture is limited to handshake metadata (SNI, version, cipher suite). The collector does not decrypt application traffic. - -:::note - -L7 logs require a minimum of 1 additional GB of log storage per node, per one-day retention period. Adjust your [Log Storage](../../../operations/logstorage/adjust-log-storage-size.mdx) before you start tasks in the next section. - -::: - -## Configure L7 logs - -Enable the eBPF L7 collector by setting the [L7ObservabilityEnabled](../../../reference/component-resources/node/felix/configuration.mdx#L7ObservabilityEnabled) field to `true` on the default `FelixConfiguration`. The setting is independent of `bpfEnabled` — the collector works with any data plane. - -```bash -kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":true}}' -``` - -To disable the collector, set the field back to `false`. Felix tears down its BPF programs and cleans up pinned maps; no node reboot is required. - -```bash -kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":false}}' -``` - -## View L7 logs - -The eBPF collector writes L7 events as JSON to a log file on each node. By default the file is at `/var/log/calico/l7logs/l7.log`. The directory is controlled by the [L7LogsFileDirectory](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileDirectory) field on `FelixConfiguration`. - -To tail the log on a node: - -```bash -tail -f /var/log/calico/l7logs/l7.log -``` - -For the JSON schema, see [L7 log data types](datatypes.mdx). diff --git a/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx b/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx deleted file mode 100644 index 173fe8a852..0000000000 --- a/calico-enterprise/observability/elastic/l7/configure-istio-waypoint.mdx +++ /dev/null @@ -1,23 +0,0 @@ ---- -description: Capture L7 logs through an Istio Ambient Mode Waypoint proxy. ---- - -# Configure L7 logs with Istio Waypoint - -:::note - -This page is a placeholder. Content for L7 log collection via Istio Ambient Mode Waypoint proxies is pending. - -::: - -## Big picture - -## Value - -## Concepts - -## Before you begin - -## Configure L7 logs - -## View L7 logs in the web console diff --git a/calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx b/calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx deleted file mode 100644 index 478adeeb37..0000000000 --- a/calico-enterprise/observability/elastic/l7/datatypes-legacy.mdx +++ /dev/null @@ -1,36 +0,0 @@ ---- -description: Reference of key/value fields that Calico Enterprise sends to Elasticsearch for L7 logs, including durations, byte counts, and HTTP request metadata. ---- - -# L7 log data types - -## Big picture - -$[prodname] sends the following data to Elasticsearch. - -The following table details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html). - -| Name | Datatype | Description | -| ------------------------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `host` | keyword | Name of the node that collected the L7 log entry. | -| `start_time` | date | Start time of log collection in Unix timestamp format. | -| `end_time` | date | End time of log collection in Unix timestamp format. | -| `bytes_in` | long | Number of incoming bytes since the last export. | -| `bytes_out` | long | Number of outgoing bytes since the last export. | -| `duration_mean` | long | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | -| `duration_max` | long | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | -| `count` | long | Number of requests that match this combination of L7 data. | -| `src_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | -| `src_namespace` | keyword | Namespace of the source endpoint. | -| `src_type` | keyword | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | -| `dest_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | -| `dest_namespace` | keyword | Namespace of the destination endpoint. | -| `dest_type` | keyword | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | -| `dest_service_name` | keyword | Name of the destination service. This may be empty if the request was not made against a service. | -| `dest_service_namespace` | keyword | Namespace of the destination service. This may be empty if the request was not made against a service. | -| `dest_service_port` | long | Destination service port. | -| `url` | keyword | URL that the request was made against. | -| `response_code` | keyword | Response code returned by the request. | -| `method` | keyword | HTTP method for the request. | -| `user_agent` | keyword | User agent of the request. | -| `type` | keyword | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | diff --git a/calico-enterprise/observability/elastic/l7/datatypes.mdx b/calico-enterprise/observability/elastic/l7/datatypes.mdx index 1e98127fa1..3c3b06b4ec 100644 --- a/calico-enterprise/observability/elastic/l7/datatypes.mdx +++ b/calico-enterprise/observability/elastic/l7/datatypes.mdx @@ -8,11 +8,11 @@ description: Reference of key/value fields that Calico Enterprise sends to Elast $[prodname] supports multiple L7 collectors that write into the same Elasticsearch index: -- **Envoy** — the legacy sidecar/proxy-based collector. See [Configure L7 logs](configure.mdx). -- **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Configure L7 logs with eBPF](configure-bpf.mdx). -- **Waypoint** — collection through an Istio Ambient Mode Waypoint proxy. See [Configure L7 logs with Istio Waypoint](configure-istio-waypoint.mdx). +- **Envoy** — the legacy sidecar/proxy-based collector. See [Enable Envoy collector](configure.mdx). +- **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Enable eBPF collector](enable-ebpf.mdx). +- **Waypoint** — collection through an Istio Ambient Mode Waypoint proxy. See [Enable Waypoint collector](enable-waypoint.mdx). -The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. A qualifier like "eBPF (TLS only)" means the field is populated only for events of that protocol family. +The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. Descriptions prefixed with _HTTP only_ or _TLS only_ mark fields that are populated for events of only that protocol family. | Name | Datatype | Populated by | Description | | ------------------------ | -------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -33,15 +33,15 @@ The table below details the key/value pairs in the JSON blob, including their [E | `dest_service_name` | keyword | All | Name of the destination service. This may be empty if the request was not made against a service. | | `dest_service_namespace` | keyword | All | Namespace of the destination service. This may be empty if the request was not made against a service. | | `dest_service_port` | long | All | Destination service port. | -| `url` | keyword | All (HTTP only) | URL that the request was made against. | -| `response_code` | keyword | All (HTTP only) | Response code returned by the request. | -| `method` | keyword | All (HTTP only) | HTTP method for the request. | -| `user_agent` | keyword | Envoy, Waypoint | User agent of the request. | -| `type` | keyword | Envoy, Waypoint | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | +| `url` | keyword | All | _HTTP only._ URL that the request was made against. | +| `response_code` | keyword | All | _HTTP only._ Response code returned by the request. | +| `method` | keyword | All | _HTTP only._ HTTP method for the request. | +| `user_agent` | keyword | All | _HTTP only._ User agent of the request. | +| `type` | keyword | All | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | | `protocol` | keyword | eBPF, Waypoint | Wire-protocol family of the captured event. Possible values: `http`, `tls`. | -| `protocol_version` | keyword | eBPF, Waypoint | Wire-protocol version. For HTTP events, possible values include `1.0` and `1.1`. | +| `protocol_version` | keyword | eBPF, Waypoint | Wire-protocol version. For HTTP events, possible values include `1.0` and `1.1`. | | `collector_type` | keyword | eBPF, Waypoint | Category of the collector that produced the event. Possible values include `ebpf` and `waypoint`. | | `collector_name` | keyword | eBPF, Waypoint | Name of the specific collector that produced the event. For the kernel TCP-layer eBPF collector, the value is `ebpf-tcp`. | -| `tls_sni` | keyword | eBPF (TLS only) | TLS Server Name Indication sent by the client in the ClientHello. | -| `tls_version` | keyword | eBPF (TLS only) | Negotiated TLS protocol version (for example, `TLS 1.2`, `TLS 1.3`). | -| `tls_cipher_suite` | keyword | eBPF (TLS only) | Negotiated TLS cipher suite (for example, `TLS_AES_256_GCM_SHA384`). | +| `tls_sni` | keyword | eBPF | _TLS only._ TLS Server Name Indication sent by the client in the ClientHello. | +| `tls_version` | keyword | eBPF | _TLS only._ Negotiated TLS protocol version (for example, `TLS 1.2`, `TLS 1.3`). | +| `tls_cipher_suite` | keyword | eBPF | _TLS only._ Negotiated TLS cipher suite (for example, `TLS_AES_256_GCM_SHA384`). | diff --git a/calico-enterprise/observability/elastic/l7/enable-ebpf.mdx b/calico-enterprise/observability/elastic/l7/enable-ebpf.mdx new file mode 100644 index 0000000000..f18573b1d8 --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/enable-ebpf.mdx @@ -0,0 +1,42 @@ +--- +description: Enable the kernel TCP-layer eBPF collector to capture HTTP and TLS L7 logs without a proxy or sidecar. +--- + +# Enable eBPF collector + +Enable the kernel TCP-layer eBPF collector to capture HTTP request/response and TLS handshake metadata without deploying a proxy or injecting sidecars. For background on what the collector does and how it compares to other options, see [L7 logs overview](overview.mdx). + +## Before you begin + +### Limitations + +* Requires Linux kernel 5.17 or later on every node. +* HTTP parsing covers HTTP/1.0 and HTTP/1.1 only, because their requests and responses travel as plain text on the wire. HTTP/2 and HTTP/3 are not captured as HTTP, and encrypted HTTP traffic surfaces only as TLS metadata. +* HTTP capture tracks only the most recent request per socket; when a single connection carries multiple back-to-back requests, some may be missed. +* TLS capture is limited to handshake metadata (SNI, version, cipher suite). The collector does not decrypt application traffic. + +## Enable the collector + +Enable the eBPF L7 collector by setting the [L7ObservabilityEnabled](../../../reference/component-resources/node/felix/configuration.mdx#L7ObservabilityEnabled) field to `true` on the default `FelixConfiguration`. The setting is independent of `bpfEnabled` — the collector works with any data plane. + +```bash +kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":true}}' +``` + +To disable the collector, set the field back to `false`. Felix tears down its BPF programs and cleans up pinned maps; no node reboot is required. + +```bash +kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":false}}' +``` + +## View L7 logs + +The eBPF collector writes L7 events as JSON to a log file on each node. By default the file is at `/var/log/calico/l7logs/l7.log`. The directory is controlled by the [L7LogsFileDirectory](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileDirectory) field on `FelixConfiguration`. + +To tail the log on a node: + +```bash +tail -f /var/log/calico/l7logs/l7.log +``` + +For the JSON schema, see [L7 log data types](datatypes.mdx). diff --git a/calico-enterprise/observability/elastic/l7/enable-waypoint.mdx b/calico-enterprise/observability/elastic/l7/enable-waypoint.mdx new file mode 100644 index 0000000000..4cdc6041cd --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/enable-waypoint.mdx @@ -0,0 +1,19 @@ +--- +description: Enable the Istio Ambient Mode Waypoint-based L7 collector. +--- + +# Enable Waypoint collector + +Enable the Istio Ambient Mode Waypoint-based L7 collector to capture HTTP/1, HTTP/2, and gRPC traffic routed through a Waypoint proxy. For background on what the collector does and how it compares to other options, see [L7 logs overview](overview.mdx). + +:::note + +This page is a placeholder. Content for L7 log collection via Istio Ambient Mode Waypoint proxies is pending. + +::: + +## Before you begin + +## Enable the collector + +## View L7 logs diff --git a/calico-enterprise/observability/elastic/l7/overview.mdx b/calico-enterprise/observability/elastic/l7/overview.mdx new file mode 100644 index 0000000000..ea458b4a1b --- /dev/null +++ b/calico-enterprise/observability/elastic/l7/overview.mdx @@ -0,0 +1,53 @@ +--- +description: Overview of Calico Enterprise L7 logs, the available collectors, and how to choose between them. +--- + +# L7 logs + +## Big picture + +$[prodname] L7 logs capture HTTP request/response and TLS handshake metadata for traffic between workloads in your cluster. They show what was actually sent (method, URL, response code, SNI, TLS version) between specific pods, going beyond the L3/4 flow log view of who-talked-to-whom. + +## Value + +Platform operators and development teams use L7 logs to: + +- See how applications interact, request by request. +- Spot anomalous behavior — attempts to access restricted URLs, unexpected user agents, scans for particular paths. +- Inspect TLS sessions by SNI, negotiated version, and cipher suite. + +L7 logs complement L3/4 flow logs, which expose only the source/destination pair without request-level detail. + +## Collectors + +$[prodname] supports multiple L7 collectors. They all feed the same aggregation and reporting pipeline; each entry on the log file is tagged with `collector_type` and `collector_name` so consumers can filter or join sources as needed. + +- **eBPF collector** — kernel TCP-layer probes that capture HTTP and TLS metadata with no data path proxy or sidecar. Enable once at the cluster level. Data-plane-independent. See [Enable eBPF collector](enable-ebpf.mdx). +- **Istio Waypoint collector** — collection through an Istio Ambient Mode Waypoint proxy. Full HTTP/1, HTTP/2, and gRPC coverage in exchange for the Waypoint deployment overhead. See [Enable Waypoint collector](enable-waypoint.mdx). +- **Envoy collector** _(legacy, deprecated)_ — sidecar Envoy injected per workload. Being phased out. See [Enable Envoy collector](configure.mdx). + +### Why more than one collector? + +Each collector lives at a different point in the request path and exposes different trade-offs. Pick by where you need coverage and what infrastructure you're willing to run: + +| | eBPF | Istio Waypoint | Envoy (legacy) | +| --- | --- | --- | --- | +| Additional infrastructure | None — runs inside Felix | Waypoint Envoy pods per namespace/service | Sidecar Envoy per workload | +| Protocol coverage | HTTP/1.x, TLS handshake | HTTP/1, HTTP/2, gRPC | HTTP | +| Opt-in granularity | Cluster-wide (FelixConfiguration) | Per namespace/service | Per workload (label + annotation) | +| Service-mesh compatibility | Any data plane | Requires Istio Ambient Mode | Not compatible with Istio | + +Multiple collectors can coexist in the same cluster when needs overlap — for example, the eBPF collector for cluster-wide observability plus the Waypoint collector for HTTP/2 in selected namespaces. + +:::note + +L7 logs require a minimum of 1 additional GB of log storage per node, per one-day retention period. Adjust your [Log Storage](../../../operations/logstorage/adjust-log-storage-size.mdx) before enabling a collector. + +::: + +## Next steps + +- [Enable eBPF collector](enable-ebpf.mdx) +- [Enable Waypoint collector](enable-waypoint.mdx) +- [L7 log aggregation](aggregation.mdx) +- [L7 log data types](datatypes.mdx) diff --git a/calico-enterprise/observability/get-started-cem.mdx b/calico-enterprise/observability/get-started-cem.mdx index 8f24a637a7..67856a421a 100644 --- a/calico-enterprise/observability/get-started-cem.mdx +++ b/calico-enterprise/observability/get-started-cem.mdx @@ -19,7 +19,7 @@ The Dashboard provides a birds-eye view of cluster activity. Note the following: - The filter panel at the top lets you change dashboard views and the time range. - The **Layout Settings** shows the default metrics. To get WireGuard metrics for pod-to-pod and host-to-host encryption, you must [enable WireGuard](../compliance/encrypt-cluster-pod-traffic.mdx). -- For application-related dashboard cards to show data, like HTTP Response Codes or Url Requests, you need to [configure L7 logs](elastic/l7/configure.mdx). +- For application-related dashboard cards to show data, like HTTP Response Codes or Url Requests, you need to [configure L7 logs](elastic/l7/overview.mdx). ![dashboards](/img/calico-enterprise/dashboards.png) diff --git a/calico-enterprise/observability/kibana.mdx b/calico-enterprise/observability/kibana.mdx index f1370d3f81..766111a95e 100644 --- a/calico-enterprise/observability/kibana.mdx +++ b/calico-enterprise/observability/kibana.mdx @@ -57,7 +57,7 @@ The dashboard provides the following metrics/data, which can be edited as requir ![l7-dashboard](/img/calico-enterprise/l7-dashboard.png) -The L7 HTTP dashboard provides application performance metrics for in-scope Kubernetes services. The data can assist service owners and platform personnel in assessing the health of cluster workloads without the need for a full service mesh. [L7 logs](elastic/l7/configure.mdx) are not enabled by default, and must be configured. +The L7 HTTP dashboard provides application performance metrics for in-scope Kubernetes services. The data can assist service owners and platform personnel in assessing the health of cluster workloads without the need for a full service mesh. [L7 logs](elastic/l7/overview.mdx) are not enabled by default, and must be configured. The default metrics are: diff --git a/sidebars-calico-enterprise.js b/sidebars-calico-enterprise.js index b5835dbc7a..8834013ecd 100644 --- a/sidebars-calico-enterprise.js +++ b/sidebars-calico-enterprise.js @@ -403,12 +403,12 @@ module.exports = { label: 'L7 logs', link: { type: 'doc', id: 'observability/elastic/l7/index' }, items: [ - { type: 'doc', id: 'observability/elastic/l7/configure-bpf', label: '+ Configure with eBPF' }, - { type: 'doc', id: 'observability/elastic/l7/configure-istio-waypoint', label: '+ Configure with Istio Waypoint' }, - { type: 'doc', id: 'observability/elastic/l7/datatypes', label: '+ Data types' }, - { type: 'doc', id: 'observability/elastic/l7/aggregation', label: '+ Aggregation' }, - 'observability/elastic/l7/configure', - { type: 'doc', id: 'observability/elastic/l7/datatypes-legacy', label: 'L7 log data types' }, + { type: 'doc', id: 'observability/elastic/l7/overview', label: 'Overview' }, + { type: 'doc', id: 'observability/elastic/l7/enable-ebpf', label: 'Enable eBPF collector' }, + { type: 'doc', id: 'observability/elastic/l7/enable-waypoint', label: 'Enable Waypoint collector' }, + { type: 'doc', id: 'observability/elastic/l7/configure', label: 'Enable Envoy collector' }, + { type: 'doc', id: 'observability/elastic/l7/datatypes', label: 'Data types' }, + { type: 'doc', id: 'observability/elastic/l7/aggregation', label: 'Aggregation' }, ], }, 'observability/elastic/troubleshoot', From 1b9cd7c0b9c2237719de766b081b5795bd97f97d Mon Sep 17 00:00:00 2001 From: Lucas Sampaio Date: Fri, 22 May 2026 11:03:26 -0700 Subject: [PATCH 5/6] Hide Istio Waypoint content from production build The Istio Waypoint collector will land in a separate PR. For now: - enable-waypoint.mdx: add draft: true; Docusaurus excludes it from production builds and auto-filters the sidebar entry - overview.mdx, datatypes.mdx: remove inline Waypoint references; leave TODO notes in single-line JSX comments so the next-PR author can find the spots - overview.mdx: drop the Istio Waypoint column from the comparison table for this PR - datatypes.mdx: change "Populated by" from "eBPF, Waypoint" to "eBPF" Co-Authored-By: Claude Opus 4.7 (1M context) --- .../observability/elastic/l7/datatypes.mdx | 10 +++++----- .../elastic/l7/enable-waypoint.mdx | 1 + .../observability/elastic/l7/overview.mdx | 20 ++++++++++--------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/calico-enterprise/observability/elastic/l7/datatypes.mdx b/calico-enterprise/observability/elastic/l7/datatypes.mdx index 3c3b06b4ec..750a460e69 100644 --- a/calico-enterprise/observability/elastic/l7/datatypes.mdx +++ b/calico-enterprise/observability/elastic/l7/datatypes.mdx @@ -10,7 +10,7 @@ $[prodname] supports multiple L7 collectors that write into the same Elasticsear - **Envoy** — the legacy sidecar/proxy-based collector. See [Enable Envoy collector](configure.mdx). - **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Enable eBPF collector](enable-ebpf.mdx). -- **Waypoint** — collection through an Istio Ambient Mode Waypoint proxy. See [Enable Waypoint collector](enable-waypoint.mdx). +{/* TODO: re-add the Waypoint collector bullet when the Waypoint PR lands. */} The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. Descriptions prefixed with _HTTP only_ or _TLS only_ mark fields that are populated for events of only that protocol family. @@ -38,10 +38,10 @@ The table below details the key/value pairs in the JSON blob, including their [E | `method` | keyword | All | _HTTP only._ HTTP method for the request. | | `user_agent` | keyword | All | _HTTP only._ User agent of the request. | | `type` | keyword | All | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | -| `protocol` | keyword | eBPF, Waypoint | Wire-protocol family of the captured event. Possible values: `http`, `tls`. | -| `protocol_version` | keyword | eBPF, Waypoint | Wire-protocol version. For HTTP events, possible values include `1.0` and `1.1`. | -| `collector_type` | keyword | eBPF, Waypoint | Category of the collector that produced the event. Possible values include `ebpf` and `waypoint`. | -| `collector_name` | keyword | eBPF, Waypoint | Name of the specific collector that produced the event. For the kernel TCP-layer eBPF collector, the value is `ebpf-tcp`. | +| `protocol` | keyword | eBPF | Wire-protocol family of the captured event. Possible values: `http`, `tls`. | +| `protocol_version` | keyword | eBPF | Wire-protocol version. For HTTP events, possible values include `1.0` and `1.1`. | +| `collector_type` | keyword | eBPF | Category of the collector that produced the event. For the kernel TCP-layer eBPF collector, the value is `ebpf`. | +| `collector_name` | keyword | eBPF | Name of the specific collector that produced the event. For the kernel TCP-layer eBPF collector, the value is `ebpf-tcp`. | | `tls_sni` | keyword | eBPF | _TLS only._ TLS Server Name Indication sent by the client in the ClientHello. | | `tls_version` | keyword | eBPF | _TLS only._ Negotiated TLS protocol version (for example, `TLS 1.2`, `TLS 1.3`). | | `tls_cipher_suite` | keyword | eBPF | _TLS only._ Negotiated TLS cipher suite (for example, `TLS_AES_256_GCM_SHA384`). | diff --git a/calico-enterprise/observability/elastic/l7/enable-waypoint.mdx b/calico-enterprise/observability/elastic/l7/enable-waypoint.mdx index 4cdc6041cd..e78bd48de5 100644 --- a/calico-enterprise/observability/elastic/l7/enable-waypoint.mdx +++ b/calico-enterprise/observability/elastic/l7/enable-waypoint.mdx @@ -1,5 +1,6 @@ --- description: Enable the Istio Ambient Mode Waypoint-based L7 collector. +draft: true --- # Enable Waypoint collector diff --git a/calico-enterprise/observability/elastic/l7/overview.mdx b/calico-enterprise/observability/elastic/l7/overview.mdx index ea458b4a1b..ebc7884f1d 100644 --- a/calico-enterprise/observability/elastic/l7/overview.mdx +++ b/calico-enterprise/observability/elastic/l7/overview.mdx @@ -23,21 +23,23 @@ L7 logs complement L3/4 flow logs, which expose only the source/destination pair $[prodname] supports multiple L7 collectors. They all feed the same aggregation and reporting pipeline; each entry on the log file is tagged with `collector_type` and `collector_name` so consumers can filter or join sources as needed. - **eBPF collector** — kernel TCP-layer probes that capture HTTP and TLS metadata with no data path proxy or sidecar. Enable once at the cluster level. Data-plane-independent. See [Enable eBPF collector](enable-ebpf.mdx). -- **Istio Waypoint collector** — collection through an Istio Ambient Mode Waypoint proxy. Full HTTP/1, HTTP/2, and gRPC coverage in exchange for the Waypoint deployment overhead. See [Enable Waypoint collector](enable-waypoint.mdx). +{/* TODO: re-add Istio Waypoint collector bullet when the Waypoint PR lands. */} - **Envoy collector** _(legacy, deprecated)_ — sidecar Envoy injected per workload. Being phased out. See [Enable Envoy collector](configure.mdx). ### Why more than one collector? Each collector lives at a different point in the request path and exposes different trade-offs. Pick by where you need coverage and what infrastructure you're willing to run: -| | eBPF | Istio Waypoint | Envoy (legacy) | -| --- | --- | --- | --- | -| Additional infrastructure | None — runs inside Felix | Waypoint Envoy pods per namespace/service | Sidecar Envoy per workload | -| Protocol coverage | HTTP/1.x, TLS handshake | HTTP/1, HTTP/2, gRPC | HTTP | -| Opt-in granularity | Cluster-wide (FelixConfiguration) | Per namespace/service | Per workload (label + annotation) | -| Service-mesh compatibility | Any data plane | Requires Istio Ambient Mode | Not compatible with Istio | +{/* TODO: re-add the Istio Waypoint column when the Waypoint PR lands. */} -Multiple collectors can coexist in the same cluster when needs overlap — for example, the eBPF collector for cluster-wide observability plus the Waypoint collector for HTTP/2 in selected namespaces. +| | eBPF | Envoy (legacy) | +| --- | --- | --- | +| Additional infrastructure | None — runs inside Felix | Sidecar Envoy per workload | +| Protocol coverage | HTTP/1.x, TLS handshake | HTTP | +| Opt-in granularity | Cluster-wide (FelixConfiguration) | Per workload (label + annotation) | +| Service-mesh compatibility | Any data plane | Not compatible with Istio | + +Multiple collectors can coexist in the same cluster, with each L7 log entry tagged by `collector_type` and `collector_name` so consumers can filter or join sources as needed. :::note @@ -48,6 +50,6 @@ L7 logs require a minimum of 1 additional GB of log storage per node, per one-da ## Next steps - [Enable eBPF collector](enable-ebpf.mdx) -- [Enable Waypoint collector](enable-waypoint.mdx) +{/* TODO: re-add [Enable Waypoint collector](enable-waypoint.mdx) when the Waypoint PR lands. */} - [L7 log aggregation](aggregation.mdx) - [L7 log data types](datatypes.mdx) From ae6947fabdee2a6d0074714f8e40f77eb5430471 Mon Sep 17 00:00:00 2001 From: Lucas Sampaio Date: Fri, 22 May 2026 16:15:40 -0700 Subject: [PATCH 6/6] Harden L7 aggregation playground and rename to L7AggregationKeyDemo Co-Authored-By: Claude Opus 4.7 (1M context) --- .../index.js | 178 +++++------------- .../L7AggregationKeyDemo/styles.module.css | 118 ++++++++++++ .../observability/elastic/l7/aggregation.mdx | 4 +- 3 files changed, 171 insertions(+), 129 deletions(-) rename calico-enterprise/_includes/components/{L7AggregationSandbox => L7AggregationKeyDemo}/index.js (63%) create mode 100644 calico-enterprise/_includes/components/L7AggregationKeyDemo/styles.module.css diff --git a/calico-enterprise/_includes/components/L7AggregationSandbox/index.js b/calico-enterprise/_includes/components/L7AggregationKeyDemo/index.js similarity index 63% rename from calico-enterprise/_includes/components/L7AggregationSandbox/index.js rename to calico-enterprise/_includes/components/L7AggregationKeyDemo/index.js index 713ed3396f..d9808138d1 100644 --- a/calico-enterprise/_includes/components/L7AggregationSandbox/index.js +++ b/calico-enterprise/_includes/components/L7AggregationKeyDemo/index.js @@ -1,4 +1,5 @@ import React, { useState, useMemo, useEffect, useRef } from 'react'; +import styles from './styles.module.css'; const SAMPLE = [ { src: 'front', dest: 'cart', svc: 'cart-svc', method: 'GET', url: '/cart', code: '200', ua: 'client', protocol: 'http', sni: null }, @@ -37,9 +38,14 @@ const COLUMNS = [ ]; const HIGHLIGHT_MS = 1500; +const FADED_ROW_OPACITY = 0.7; +const DATA_COL_WIDTH = `${90 / COLUMNS.length}%`; +const COUNT_COL_WIDTH = '10%'; -export default function L7AggregationSandbox() { - const [checked, setChecked] = useState( +const cx = (...names) => names.filter(Boolean).join(' '); + +export default function L7AggregationKeyDemo() { + const [checked, setChecked] = useState(() => Object.fromEntries(AGGREGATORS.map((a) => [a.id, true])), ); @@ -107,8 +113,11 @@ export default function L7AggregationSandbox() { }); }; - const visibleRows = displayedRows.filter( - (row) => row.__primary || expanded.has(row.primaryOrigIdx), + const visibleRows = useMemo( + () => displayedRows.filter( + (row) => row.__primary || expanded.has(row.primaryOrigIdx), + ), + [displayedRows, expanded], ); useEffect(() => { @@ -133,72 +142,31 @@ export default function L7AggregationSandbox() { } } prevRef.current = displayedRows; + if (newHL.size === 0 && highlights.size === 0) return; setHighlights(newHL); if (newHL.size === 0) return; const t = setTimeout(() => setHighlights(new Map()), HIGHLIGHT_MS); return () => clearTimeout(t); + // eslint-disable-next-line react-hooks/exhaustive-deps }, [displayedRows]); const toggle = (id) => setChecked((c) => ({ ...c, [id]: !c[id] })); - const ROW_HEIGHT = '3.2em'; - const baseCellStyle = { - padding: '4px 8px', - fontFamily: 'var(--ifm-font-family-monospace)', - fontSize: '0.85em', - overflowWrap: 'anywhere', - verticalAlign: 'top', - transition: 'background-color 1s ease', - height: ROW_HEIGHT, - boxSizing: 'border-box', - backgroundColor: 'transparent', - }; - const cellStyleFor = (rowIdx, key) => { + const cellClassFor = (rowIdx, key, extra) => { const direction = highlights.get(`${rowIdx}:${key}`); - if (direction === 'add') { - return { - ...baseCellStyle, - backgroundColor: 'var(--ifm-color-success-contrast-background)', - }; - } - if (direction === 'remove') { - return { - ...baseCellStyle, - backgroundColor: 'var(--ifm-color-danger-contrast-background)', - }; - } - return baseCellStyle; - }; - const headerStyle = { - ...baseCellStyle, - whiteSpace: 'normal', - wordBreak: 'break-word', - transition: 'none', + return cx( + styles.cell, + direction === 'add' && styles.cellAdd, + direction === 'remove' && styles.cellRemove, + extra, + ); }; return (
-
+
{AGGREGATORS.map((a) => ( -
-
- - +
+ + added (value gained or count up) - - + + removed (value lost or count down)
-
- +
+
- + {COLUMNS.map((c) => ( - + ))} - + {COLUMNS.map((c) => ( - + ))} @@ -265,49 +207,26 @@ export default function L7AggregationSandbox() { const prevRow = visibleRows[displayIdx - 1]; const isBucketStart = !prevRow || prevRow.bucketIdx !== row.bucketIdx; - const borderTop = isBucketStart && displayIdx > 0 - ? '2px solid var(--ifm-color-emphasis-300)' - : undefined; - const styleForCell = (key) => { - const base = cellStyleFor(row.originalIdx, key); - return borderTop ? { ...base, borderTop } : base; - }; - const rowOpacity = row.__primary ? 1 : 0.7; + const bucketStartClass = isBucketStart && displayIdx > 0 + ? styles.bucketStart + : null; const isExpandable = row.__primary && row.bucketSize > 1; const isOpen = expanded.has(row.originalIdx); return ( - {COLUMNS.map((c) => ( - + ))} ); diff --git a/calico-enterprise/_includes/components/L7AggregationKeyDemo/styles.module.css b/calico-enterprise/_includes/components/L7AggregationKeyDemo/styles.module.css new file mode 100644 index 0000000000..f3c7bb959e --- /dev/null +++ b/calico-enterprise/_includes/components/L7AggregationKeyDemo/styles.module.css @@ -0,0 +1,118 @@ +.controls { + display: flex; + flex-wrap: wrap; + gap: 8px 16px; + margin: 12px 0; + padding: 12px; + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 4px; +} + +.controlLabel { + display: inline-flex; + align-items: center; + gap: 4px; + cursor: pointer; +} + +.legend { + display: flex; + gap: 16px; + margin: 8px 0; + font-size: 0.85em; + opacity: 0.8; +} + +.legendItem { + display: inline-flex; + align-items: center; + gap: 4px; +} + +.legendSwatch { + display: inline-block; + width: 0.9em; + height: 0.9em; + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 2px; +} + +.legendSwatchAdd { + background-color: var(--ifm-color-success-contrast-background); +} + +.legendSwatchRemove { + background-color: var(--ifm-color-danger-contrast-background); +} + +.tableWrap { + overflow-x: auto; +} + +.table { + table-layout: fixed; + width: 100%; +} + +.row { + background-color: transparent; + transition: opacity 0.4s ease; +} + +.cell { + padding: 4px 8px; + font-family: var(--ifm-font-family-monospace); + font-size: 0.85em; + overflow-wrap: anywhere; + vertical-align: top; + transition: background-color 1s ease; + height: 3.2em; + box-sizing: border-box; + background-color: transparent; +} + +.header { + white-space: normal; + word-break: break-word; + transition: none; +} + +.cellAdd { + background-color: var(--ifm-color-success-contrast-background); +} + +.cellRemove { + background-color: var(--ifm-color-danger-contrast-background); +} + +.bucketStart { + border-top: 2px solid var(--ifm-color-emphasis-300); +} + +.expandBtn { + display: inline-flex; + align-items: center; + justify-content: center; + width: 1.5em; + height: 1.5em; + margin-right: 6px; + padding: 0; + background: var(--ifm-color-emphasis-100); + border: 1px solid var(--ifm-color-emphasis-400); + border-radius: 3px; + cursor: pointer; + color: var(--ifm-color-emphasis-800); + font-family: inherit; + font-size: 1em; + font-weight: 700; + line-height: 1; +} + +.expandBtn:hover { + background: var(--ifm-color-emphasis-200); +} + +.expandBtn:focus-visible { + outline: 2px solid var(--ifm-color-primary); + outline-offset: 1px; +} diff --git a/calico-enterprise/observability/elastic/l7/aggregation.mdx b/calico-enterprise/observability/elastic/l7/aggregation.mdx index d5062f3c16..8a857fc325 100644 --- a/calico-enterprise/observability/elastic/l7/aggregation.mdx +++ b/calico-enterprise/observability/elastic/l7/aggregation.mdx @@ -2,7 +2,7 @@ description: Control how Calico Enterprise aggregates L7 log entries by selecting which fields participate in the aggregation key. --- -import L7AggregationSandbox from '../../../_includes/components/L7AggregationSandbox'; +import L7AggregationKeyDemo from '../../../_includes/components/L7AggregationKeyDemo'; # L7 log aggregation @@ -16,7 +16,7 @@ Including a field makes the logs more granular (more rows, fewer requests per ro The table below shows a small set of sample L7 events. Each visible row is what would be written to the L7 log, with `count` equal to the number of requests merged into it. Toggle the checkboxes to include or exclude each field from the aggregation key. - + ## Aggregation fields
countcount{c.label}{c.label}
+ {isExpandable ? ( {row[c.key]} + {row[c.key]} +