diff --git a/.config/mise/tasks/semconv/check b/.config/mise/tasks/semconv/check index 55edb693fe..9408a4acdb 100755 --- a/.config/mise/tasks/semconv/check +++ b/.config/mise/tasks/semconv/check @@ -10,3 +10,11 @@ trap 'rm -rf "$generated"' EXIT HUP INT TERM "$root/.config/mise/tasks/semconv/generate" "$generated" diff -ru "$expected" "$generated" + +if grep -n '"eventstore-' \ + "$root/src/EventStore.Core/MetricsBootstrapper.cs" \ + "$root/src/EventStore.Projections.Core/ProjectionsSubsystem.cs" +then + echo "Built-in metric names must come from the generated MetricNames catalog." >&2 + exit 1 +fi diff --git a/.config/mise/tasks/semconv/generate b/.config/mise/tasks/semconv/generate index f808e5479e..b0ce43f4c1 100755 --- a/.config/mise/tasks/semconv/generate +++ b/.config/mise/tasks/semconv/generate @@ -6,10 +6,13 @@ set -eu root=$(CDPATH='' cd -- "$(dirname -- "$0")/../../../.." && pwd) output=${1:-"$root/src/TrogonEventStore.SemanticConventions/Generated"} registry_version=$(sed -n '1p' "$root/otel/semconv/registry-version") -registry="https://github.com/open-telemetry/semantic-conventions@${registry_version}[model]" +registry="$root/otel/semconv/registry" +official_registry="https://github.com/open-telemetry/semantic-conventions@${registry_version}[model]" staging=$(mktemp -d) trap 'rm -rf "$staging"' EXIT HUP INT TERM +grep -Fqx " registry_path: $official_registry" "$registry/manifest.yaml" + weaver registry check \ --future \ --registry "$registry" @@ -17,7 +20,18 @@ weaver registry check \ weaver registry generate csharp "$staging" \ --future \ --registry "$registry" \ - --templates "$root/otel/semconv/templates" + --templates "$root/otel/semconv/templates" \ + -D attributes=false \ + -D custom_attributes=true \ + -D official_metrics=false + +weaver registry generate csharp "$staging" \ + --future \ + --registry "$official_registry" \ + --templates "$root/otel/semconv/templates" \ + -D custom_attributes=false \ + -D metrics=false \ + -D official_metrics=true set -- "$staging"/*.g.cs [ -e "$1" ] || exit 1 diff --git a/docs/diagnostics/metrics.md b/docs/diagnostics/metrics.md index 63f8c50551..ae011b7174 100644 --- a/docs/diagnostics/metrics.md +++ b/docs/diagnostics/metrics.md @@ -1,604 +1,174 @@ -# Metrics + -TrogonEventStore collects metrics in [Prometheus format](https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format), available on the `/-/metrics` endpoint. Prometheus can be configured to scrape this endpoint directly. The metrics are configured in `metricsconfig.json`. +# Metrics reference -In addition, TrogonEventStore can actively export metrics to a specified endpoint using the [OpenTelemetry Protocol](https://opentelemetry.io/docs/specs/otel/protocol/) (OTLP). +TrogonEventStore exposes metrics through the OpenTelemetry Protocol (OTLP) and the Prometheus-compatible `/-/metrics` endpoint. The canonical instrument identity is the dotted OpenTelemetry name documented here. Prometheus exporters may translate dots, units, or other name components according to their configured translation strategy. -The built-in Core and Projections meter sources are always registered. Use the `Meters` array in `metricsconfig.json` only when additional components expose their own `System.Diagnostics.Metrics` meter sources. +Configure node-specific metrics in `metricsconfig.json`. The `Meters` array adds third-party `System.Diagnostics.Metrics` meter sources; the built-in sources do not need to be listed there. -## Metrics reference +Units use UCUM notation: `s` is seconds, `By` is bytes, and `1` is dimensionless. Units such as `{event}` are annotated counts. The unit is metadata and is not repeated in the canonical instrument name. -### Caches +Telemetry resources include service identity, process creation time and PID, executable and runtime identity, and host name and architecture. Deployments should provide `host.id` through `OTEL_RESOURCE_ATTRIBUTES` when a stable machine or cloud instance identifier is available. -#### Cache hits and misses +## Built-in meter sources -TrogonEventStore tracks cache hits/misses metrics for `stream-info` and `chunk` caches. +The meter provider always subscribes to these sources: -| Time series | Type | Description | -|:---------------------------------------------------------------------------|:-------------------------|:----------------------------------------| -| `eventstore_cache_hits_misses{cache=,kind=<"hits"\|"misses">}` | [Counter](#common-types) | Total hits/misses on _CACHE_NAME_ cache | +| Meter | Source | +| --- | --- | +| `EventStore.Core` | Core TrogonEventStore instruments | +| `EventStore.Projections.Core` | Projection instruments | +| `System.Runtime` | Native .NET process, garbage collection, exception, lock, and thread pool instruments | +| `Microsoft.AspNetCore.Server.Kestrel` | Native Kestrel connection and server instruments | -Example configuration: -```json -"CacheHitsMisses": { - "StreamInfo": true, - "Chunk": false -} -``` - -Example output: -``` -# TYPE eventstore_cache_hits_misses counter -eventstore_cache_hits_misses{cache="stream-info",kind="hits"} 104329 1688157489545 -eventstore_cache_hits_misses{cache="stream-info",kind="misses"} 117 1688157489545 -``` - -#### Dynamic cache resources - -Certain caches that TrogonEventStore uses are dynamic in nature i.e. their capacity scales up/down during their lifetime. TrogonEventStore records metrics for resources being used by each such dynamic cache. - -| Time series | Type | Description | -|:---------------------------------------------------------------------------------|:-----------------------|:-----------------------------------------------------| -| `eventstore_cache_resources_bytes{cache=,kind=<"capacity"\|"size">}` | [Gauge](#common-types) | Current capacity/size of _CACHE_NAME_ cache in bytes | -| `eventstore_cache_resources_entries{cache=,kind="count"}` | [Gauge](#common-types) | Current number of entries in _CACHE_NAME_ cache | - -Example configuration: -```json -"CacheResources": true -``` - -Example output: -``` -# TYPE eventstore_cache_resources_bytes gauge -# UNIT eventstore_cache_resources_bytes bytes -eventstore_cache_resources_bytes{cache="LastEventNumber",kind="capacity"} 50000000 1688157491029 -eventstore_cache_resources_bytes{cache="LastEventNumber",kind="size"} 15804 1688157491029 - -# TYPE eventstore_cache_resources_entries gauge -# UNIT eventstore_cache_resources_entries entries -eventstore_cache_resources_entries{cache="LastEventNumber",kind="count"} 75 1688157491029 -``` - -### Checkpoints - -| Time series | Type | Description | -|:--------------------------------------------------------------------|:-----------------------|:---------------------------------------| -| `eventstore_checkpoints{name=,read="non-flushed"}` | [Gauge](#common-types) | Value for _CHECKPOINT_NAME_ checkpoint | - -Example configuration: -```json -"Checkpoints": { - "Replication": true, - "Chaser": false, - "Epoch": false, - "Index": false, - "Proposal": false, - "Truncate": false, - "Writer": false, - "StreamExistenceFilter": false -} -``` - -Example output: -``` -# TYPE eventstore_checkpoints gauge -eventstore_checkpoints{name="replication",read="non-flushed"} 613363 1688054162478 -``` - -### Elections Count - -This metric tracks the number of elections that have been completed. - -| Time series | Type | Description | -|:-----------------------------|:-------------------------|:-----------------------------| -| `eventstore_elections_count` | [Counter](#common-types) | Elections count in a cluster | - -Example configuration: -```json -"ElectionsCount": true -``` - -Example output: -``` -# TYPE eventstore_elections_count counter -eventstore_elections_count 0 1710188996949 -``` - -### Events - -These metrics track events written to and read from the server, including reads from caches. - -| Time series | Type | Description | -|:-----------------------------------------------------|:-------------------------|:--------------------| -| `eventstore_io_bytes{activity="read"}` | [Counter](#common-types) | Event bytes read | -| `eventstore_io_events{activity=<"read"\|"written">}` | [Counter](#common-types) | Events read/written | - -Example configuration: -```json -"Events": { - "Read": false, - "Written": true -} -``` - -Example output: -``` -# TYPE eventstore_io_events counter -# UNIT eventstore_io_events events -eventstore_io_events{activity="written"} 320 1687963622074 -``` - -### Gossip - -Measures the round trip latency and processing time of gossip. -Usually a node pushes new gossip to other nodes periodically or when its view of the cluster changes. Sometimes nodes pull gossip from each other if there is a suspected network problem. - -#### Gossip latency - -| Time series | Type | Description | -|:--------------------------------------------------------------------------------------------------------------------|:---------------------------|:---------------------------------------------------------------------------------------------| -| `eventstore_gossip_latency_seconds_bucket{activity="pull-from-peer",status=<"successful"\|"failed">,le=}` | [Histogram](#common-types) | Number of gossips pulled from peers with latency less than or equal to _DURATION_ in seconds | -| `eventstore_gossip_latency_seconds_bucket{activity="push-to-peer",status=<"successful"\|"failed">,le=}` | [Histogram](#common-types) | Number of gossips pushed to peers with latency less than or equal to _DURATION_ in seconds | - -#### Gossip processing - -| Time Series | Type | Description | -|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------|:-------------------------------------------------------------------------------------------------------------| -| `eventstore_gossip_processing_duration_seconds_bucket{`
`activity="push-from-peer",`
`status=<"successful"\|"failed">,`
`le=}` | [Histogram](#common-types) | Number of gossips pushed from peers that took less than or equal to _DURATION_ in seconds to process | -| `eventstore_gossip_processing_duration_seconds_bucket{`
`activity="request-from-peer",`
`status=<"successful"\|"failed">,`
`le=}` | [Histogram](#common-types) | Number of gossip requests from peers that took less than or equal to _DURATION_ in seconds to process | -| `eventstore_gossip_processing_duration_seconds_bucket{`
`activity="request-from-grpc-client",`
`status=<"successful"\|"failed">,`
`le=}` | [Histogram](#common-types) | Number of gossip requests from gRPC clients that took less than or equal to _DURATION_ in seconds to process | - -Example configuration: -```json -"Gossip": { - "PullFromPeer": false, - "PushToPeer": true, - "ProcessingPushFromPeer": false, - "ProcessingRequestFromPeer": false, - "ProcessingRequestFromGrpcClient": false -} -``` - -Example output: -``` -# TYPE eventstore_gossip_latency_seconds histogram -# UNIT eventstore_gossip_latency_seconds seconds -eventstore_gossip_latency_seconds_bucket{activity="push-to-peer",status="successful",le="0.005"} 8 1687972306948 -``` - -### Incoming gRPC calls - -| Time series | Type | Description | -|:------------------------------------------------------------------|:-------------------------|:-------------------------------------------------------------------------------------------| -| `eventstore_current_incoming_grpc_calls` | [Gauge](#common-types) | Inflight gRPC calls i.e. gRPC requests that have started on the server but not yet stopped | -| `eventstore_incoming_grpc_calls{kind="total"}` | [Counter](#common-types) | Total gRPC requests served | -| `eventstore_incoming_grpc_calls{kind="failed"}` | [Counter](#common-types) | Total gRPC requests failed | -| `eventstore_incoming_grpc_calls{`
`kind="unimplemented"}` | [Counter](#common-types) | Total gRPC requests made to unimplemented methods | -| `eventstore_incoming_grpc_calls{`
`kind="deadline-exceeded"}` | [Counter](#common-types) | Total gRPC requests for which deadline have exceeded | - -Example configuration: -```json -"IncomingGrpcCalls": { - "Current": true, - "Total": false, - "Failed": true, - "Unimplemented": false, - "DeadlineExceeded": false -} -``` - -Example output: -``` -# TYPE eventstore_current_incoming_grpc_calls gauge -eventstore_current_incoming_grpc_calls 1 1687963622074 - -# TYPE eventstore_incoming_grpc_calls counter -eventstore_incoming_grpc_calls{kind="failed"} 1 1687962877623 -``` - -#### Client protocol gRPC methods - -In addition, TrogonEventStore also records metrics for each of client protocol gRPC methods: `StreamRead`, `StreamAppend`, `StreamBatchAppend`, `StreamDelete` and `StreamTombstone`. They are grouped together according to the mapping defined in the configuration. - -| Time series | Type | Description | -|:---------------------------------------------------------------------------------------------------------------|:---------------------------|:-------------------------------------------------------------------------------------------------| -| `eventstore_grpc_method_duration_seconds_bucket{`
`activity=