feat: Add authenticated environment attributes to relay.auth span and metrics

docs/metrics.md

@@ -19,12 +19,13 @@ The Relay Proxy can export metrics via [OpenTelemetry Protocol (OTLP)](https://o
 
 ## Attributes
 
-All metrics include the following attributes:
+All metrics include the following attributes. When OpenTelemetry tracing is enabled, `environment.name` and `environment.id` are also set on the request span of every authenticated request, using these same keys, so traces and metrics can be correlated by environment.
 
 | Attribute | Description |
 |-----------|-------------|
 | `relay.id` | A unique identifier for this Relay Proxy instance, generated at startup. |
-| `environment.name` | The name of the LaunchDarkly environment as configured in the Relay Proxy. In automatic configuration or offline mode, this is the actual project and environment name from LaunchDarkly. Example: `MyApplication Staging` |
+| `environment.name` | The name of the LaunchDarkly environment as configured in the Relay Proxy. In automatic configuration or offline mode, this is the actual project and environment name from LaunchDarkly. If the environment is renamed there, subsequent metrics are reported under the new name, which starts a new series. Example: `MyApplication Staging` |
+| `environment.id` | The LaunchDarkly environment ID, when the environment is configured with a client-side environment ID. It is omitted for environments configured with only an SDK key or mobile key. Environments that differ only by payload filter share one environment ID, so group by `environment.name` to tell filtered variants apart. Example: `507f1f77bcf86cd79943902a` |
 | `platform.category` | The kind of SDK that generated the metric: `server`, `mobile`, or `browser`. |
 | `user_agent` | The user agent of the SDK making the request. Example: `Node/3.4.0` |
 | `sdk.wrapper` | The SDK wrapper identifier, if provided. Example: `flutter-client/2.0.0` |

internal/metrics/constants.go

@@ -4,6 +4,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/launchdarkly/ld-relay/v9/internal/tracing"
+
 	"go.opentelemetry.io/otel/attribute"
 	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
 )
@@ -26,12 +28,18 @@ const (
 	ServerPlatformCategory  = "server"
 )
 
+// The environment attribute keys are shared with the tracing package, which sets the same two
+// attributes on the request span.
+const (
+	envNameAttrKey = tracing.EnvNameKey
+	envIDAttrKey   = tracing.EnvIDKey
+)
+
 var (
 	relayIDAttrKey            = attribute.Key("relay.id")            //nolint:gochecknoglobals
 	platformCategoryAttrKey   = attribute.Key("platform.category")   //nolint:gochecknoglobals
 	userAgentAttrKey          = attribute.Key("user_agent")          //nolint:gochecknoglobals
 	sdkWrapperAttrKey         = attribute.Key("sdk.wrapper")         //nolint:gochecknoglobals
-	envNameAttrKey            = attribute.Key("environment.name")    //nolint:gochecknoglobals
 	applicationIDAttrKey      = attribute.Key("application.id")      //nolint:gochecknoglobals
 	applicationVersionAttrKey = attribute.Key("application.version") //nolint:gochecknoglobals
 	instanceIDAttrKey         = attribute.Key("instance.id")         //nolint:gochecknoglobals
@@ -47,7 +55,8 @@ var (
 )
 
 // buildRequestAttributes creates an OTel attribute set for request metrics using semconv attribute names
-// where applicable. All string values should be pre-sanitized via sanitizeTagValue before calling this function.
+// where applicable. All string values should be pre-sanitized via tracing.SanitizeAttributeValue before
+// calling this function.
 func buildRequestAttributes(baseKVs []attribute.KeyValue, platform, userAgent, sdkWrapper, route, method, urlScheme, applicationID, applicationVersion, instanceID string) attribute.Set {
 	attrs := make([]attribute.KeyValue, len(baseKVs), len(baseKVs)+9)
 	copy(attrs, baseKVs)
@@ -93,18 +102,8 @@ func buildDurationAttributes(baseKVs []attribute.KeyValue, platform, userAgent,
 	return attribute.NewSet(attrs...)
 }
 
-// sanitizeTagValue ensures attribute values are valid.
-// Empty values are replaced with descriptive defaults, and slashes are replaced with underscores.
-// This is appropriate for user agent strings and SDK wrapper names, but not for routes.
-func sanitizeTagValue(v string) string {
-	if strings.TrimSpace(v) == "" {
-		return "not-provided"
-	}
-	return strings.ReplaceAll(v, "/", "_")
-}
-
 // sanitizeRouteValue ensures route attribute values are valid.
-// Empty values are replaced with descriptive defaults. Unlike sanitizeTagValue,
+// Empty values are replaced with descriptive defaults. Unlike tracing.SanitizeAttributeValue,
 // slashes are preserved since they are meaningful in route paths.
 func sanitizeRouteValue(v string) string {
 	if strings.TrimSpace(v) == "" {

internal/metrics/measures.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"time"
 
+	"github.com/launchdarkly/ld-relay/v9/internal/tracing"
+
 	ldevents "github.com/launchdarkly/go-sdk-events/v3"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
@@ -125,13 +127,13 @@ type RequestInfo struct {
 }
 
 func (ri RequestInfo) sanitized() (ua, wrapper, route, method, appID, appVersion, instanceID string) {
-	return sanitizeTagValue(ri.UserAgent),
-		sanitizeTagValue(ri.SDKWrapper),
+	return tracing.SanitizeAttributeValue(ri.UserAgent),
+		tracing.SanitizeAttributeValue(ri.SDKWrapper),
 		sanitizeRouteValue(ri.Route),
-		sanitizeTagValue(ri.Method),
-		sanitizeTagValue(ri.ApplicationID),
-		sanitizeTagValue(ri.ApplicationVersion),
-		sanitizeTagValue(ri.InstanceID)
+		tracing.SanitizeAttributeValue(ri.Method),
+		tracing.SanitizeAttributeValue(ri.ApplicationID),
+		tracing.SanitizeAttributeValue(ri.ApplicationVersion),
+		tracing.SanitizeAttributeValue(ri.InstanceID)
 }
 
 // WithGauge increments the specified metric before running the function and then decrements it (for use with
@@ -143,7 +145,7 @@ func WithGauge(em *EnvironmentManager, instruments *Instruments, ri RequestInfo,
 	}
 
 	ua, wrapper, route, method, appID, appVersion, instanceID := ri.sanitized()
-	attrs := buildRequestAttributes(em.envKVs, measure.platformCategory, ua, wrapper, route, method, ri.URLScheme, appID, appVersion, instanceID)
+	attrs := buildRequestAttributes(em.attributes().kvs, measure.platformCategory, ua, wrapper, route, method, ri.URLScheme, appID, appVersion, instanceID)
 
 	if instruments != nil {
 		instruments.connections.Add(context.Background(), 1, metric.WithAttributeSet(attrs))
@@ -174,7 +176,7 @@ func RecordEventsReceivedBytes(ctx context.Context, instruments *Instruments, em
 		return
 	}
 	ua, wrapper, route, method, appID, appVersion, instanceID := ri.sanitized()
-	attrs := buildRequestAttributes(em.envKVs, platformCategory, ua, wrapper, route, method, ri.URLScheme, appID, appVersion, instanceID)
+	attrs := buildRequestAttributes(em.attributes().kvs, platformCategory, ua, wrapper, route, method, ri.URLScheme, appID, appVersion, instanceID)
 	instruments.eventsReceivedBytes.Add(ctx, bytes, metric.WithAttributeSet(attrs))
 }
 
@@ -185,7 +187,7 @@ func RecordRequestDuration(ctx context.Context, instruments *Instruments, em *En
 		return
 	}
 	ua, wrapper, route, method, appID, appVersion, instanceID := ri.sanitized()
-	attrs := buildDurationAttributes(em.envKVs, measure.platformCategory, ua, wrapper, route, method, appID, appVersion, instanceID, ri.URLScheme, ri.ProtocolVersion, ri.ErrorType, ri.StatusCode)
+	attrs := buildDurationAttributes(em.attributes().kvs, measure.platformCategory, ua, wrapper, route, method, appID, appVersion, instanceID, ri.URLScheme, ri.ProtocolVersion, ri.ErrorType, ri.StatusCode)
 	instruments.requestDuration.Record(ctx, duration.Seconds(), metric.WithAttributeSet(attrs))
 }
 
@@ -195,40 +197,39 @@ func RecordRequestDuration(ctx context.Context, instruments *Instruments, em *En
 // detached from any specific HTTP request context.
 type EventMetricsRecorder struct {
 	instruments *Instruments
-	envKVs      []attribute.KeyValue // private copy, safe for concurrent read
-	envAttrs    attribute.Set        // pre-computed to avoid concurrent sort in attribute.NewSet
+	env         *EnvironmentManager // attributes are read per record, so a rename is picked up
 }
 
 // RecordDroppedEvents records the number of events dropped due to capacity overflow.
 func (r *EventMetricsRecorder) RecordDroppedEvents(count int) {
 	if r.instruments == nil || count <= 0 {
 		return
 	}
-	r.instruments.eventsDropped.Add(context.Background(), int64(count), metric.WithAttributeSet(r.envAttrs))
+	r.instruments.eventsDropped.Add(context.Background(), int64(count), metric.WithAttributeSet(r.env.attributes().set))
 }
 
 // RecordEventsSent records the number of events successfully delivered to the events service.
 func (r *EventMetricsRecorder) RecordEventsSent(count int) {
 	if r.instruments == nil || count <= 0 {
 		return
 	}
-	r.instruments.eventsSent.Add(context.Background(), int64(count), metric.WithAttributeSet(r.envAttrs))
+	r.instruments.eventsSent.Add(context.Background(), int64(count), metric.WithAttributeSet(r.env.attributes().set))
 }
 
 // RecordPendingEvents records the current number of events pending delivery.
 func (r *EventMetricsRecorder) RecordPendingEvents(depth int) {
 	if r.instruments == nil {
 		return
 	}
-	r.instruments.pendingEvents.Record(context.Background(), int64(depth), metric.WithAttributeSet(r.envAttrs))
+	r.instruments.pendingEvents.Record(context.Background(), int64(depth), metric.WithAttributeSet(r.env.attributes().set))
 }
 
 // RecordEventsBytesSent records the size of event payloads successfully delivered.
 func (r *EventMetricsRecorder) RecordEventsBytesSent(bytes int) {
 	if r.instruments == nil || bytes <= 0 {
 		return
 	}
-	r.instruments.eventsBytesSent.Add(context.Background(), int64(bytes), metric.WithAttributeSet(r.envAttrs))
+	r.instruments.eventsBytesSent.Add(context.Background(), int64(bytes), metric.WithAttributeSet(r.env.attributes().set))
 }
 
 // RecordEventsFailedSend records the number of events that could not be delivered after all retries.
@@ -237,8 +238,9 @@ func (r *EventMetricsRecorder) RecordEventsFailedSend(count int, metadata ldeven
 	if r.instruments == nil || count <= 0 {
 		return
 	}
-	kvs := make([]attribute.KeyValue, len(r.envKVs), len(r.envKVs)+1)
-	copy(kvs, r.envKVs)
+	envKVs := r.env.attributes().kvs
+	kvs := make([]attribute.KeyValue, len(envKVs), len(envKVs)+1)
+	copy(kvs, envKVs)
 	kvs = append(kvs, statusCodeAttrKey.Int(metadata.StatusCode))
 	attrs := attribute.NewSet(kvs...)
 	r.instruments.eventsFailedSend.Add(context.Background(), int64(count), metric.WithAttributeSet(attrs))

internal/metrics/metrics.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"log/slog"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/launchdarkly/ld-relay/v9/config"
@@ -51,13 +52,41 @@ type shutdown struct {
 	closed chan struct{}
 }
 
+// envAttributes is an immutable snapshot of an environment's metric attributes. Renaming an
+// environment replaces the whole snapshot rather than mutating it, so readers need no
+// synchronization beyond the atomic load that hands them one.
+type envAttributes struct {
+	kvs []attribute.KeyValue
+	set attribute.Set
+}
+
+func newEnvAttributes(relayID, envName, envID string) *envAttributes {
+	kvs := []attribute.KeyValue{
+		relayIDAttrKey.String(relayID),
+		envNameAttrKey.String(tracing.SanitizeAttributeValue(envName)),
+	}
+	if envID != "" {
+		kvs = append(kvs, envIDAttrKey.String(envID))
+	}
+	// NewSet sorts kvs in place; doing it here, before the snapshot is published, means every
+	// later reader gets an already-sorted slice it only ever copies from.
+	return &envAttributes{kvs: kvs, set: attribute.NewSet(kvs...)}
+}
+
 // EnvironmentManager controls the metrics exporter activity for a specific LD environment.
 type EnvironmentManager struct {
-	envKVs    []attribute.KeyValue
+	relayID   string
+	envID     string
+	attrs     atomic.Pointer[envAttributes]
 	collector *RelayMetricsCollector
 	closeOnce sync.Once
 }
 
+// attributes returns the current attribute snapshot for this environment.
+func (em *EnvironmentManager) attributes() *envAttributes {
+	return em.attrs.Load()
+}
+
 // NewManager creates a Manager instance.
 func NewManager(
 	otlpConfig config.OpenTelemetryConfig,
@@ -228,28 +257,25 @@ func (m *Manager) Close() {
 }
 
 // AddEnvironment creates a new EnvironmentManager with its own attribute set that includes
-// the environment name.
-func (m *Manager) AddEnvironment(envName string, publisher events.EventPublisher) (*EnvironmentManager, error) {
+// the environment name, and the environment ID when one is available.
+func (m *Manager) AddEnvironment(envName, envID string, publisher events.EventPublisher) (*EnvironmentManager, error) {
 	m.lock.Lock()
 	defer m.lock.Unlock()
 	if m.closed {
 		return nil, errAddEnvironmentAfterClosed
 	}
 
-	envKVs := []attribute.KeyValue{
-		relayIDAttrKey.String(m.metricsRelayID),
-		envNameAttrKey.String(sanitizeTagValue(envName)),
-	}
-
 	var collector *RelayMetricsCollector
 	if publisher != nil {
 		collector = newRelayMetricsCollector(m.metricsRelayID, envName, publisher, m.flushInterval, m.logger)
 	}
 
 	em := &EnvironmentManager{
-		envKVs:    envKVs,
+		relayID:   m.metricsRelayID,
+		envID:     envID,
 		collector: collector,
 	}
+	em.attrs.Store(newEnvAttributes(m.metricsRelayID, envName, envID))
 	m.environments = append(m.environments, em)
 	return em, nil
 }
@@ -282,24 +308,31 @@ func (m *Manager) RemoveEnvironmentForUsage(envName string) {
 	m.usageChan <- removeEnvironment{envName: envName}
 }
 
+// SetEnvironmentName rebuilds this environment's metric attributes around a new environment name.
+// Relay calls this when an environment is renamed upstream, in automatic configuration or offline
+// mode, so that metrics track the rename the same way spans do. Data points recorded afterwards
+// carry the new name, which means the backend sees a new time series and the old one goes stale.
+//
+// The environment ID cannot change for a live environment, so it is carried over.
+func (em *EnvironmentManager) SetEnvironmentName(envName string) {
+	em.attrs.Store(newEnvAttributes(em.relayID, envName, em.envID))
+}
+
 // GetAttributes returns the attribute set for this EnvironmentManager.
 func (em *EnvironmentManager) GetAttributes() attribute.Set {
-	return attribute.NewSet(em.envKVs...)
+	return em.attributes().set
 }
 
 // NewEventMetricsRecorder creates an EventMetricsRecorder that records event processing metrics
 // with this environment's attributes. The returned recorder satisfies the EventMetrics interfaces
 // defined in both the events package and go-sdk-events.
 //
-// The recorder makes a private copy of the environment attributes to avoid data races with
-// attribute.NewSet's in-place sort.
+// The recorder reads the environment's attributes when it records rather than capturing them here,
+// so event metrics follow a rename like every other metric does.
 func (em *EnvironmentManager) NewEventMetricsRecorder(instruments *Instruments) *EventMetricsRecorder {
-	envKVsCopy := make([]attribute.KeyValue, len(em.envKVs))
-	copy(envKVsCopy, em.envKVs)
 	return &EventMetricsRecorder{
 		instruments: instruments,
-		envKVs:      envKVsCopy,
-		envAttrs:    attribute.NewSet(envKVsCopy...),
+		env:         em,
 	}
 }