diff --git a/pkg/controller/builder/builder.go b/pkg/controller/builder/builder.go index f264de3..71e2e23 100644 --- a/pkg/controller/builder/builder.go +++ b/pkg/controller/builder/builder.go @@ -120,6 +120,7 @@ func Build(ctx context.Context, conf Configuration, optsFuncs ...FuncOption) (*c GracefulShutdownTimeout: opts.gracefulShutdownTimeout, TracerProvider: opts.tracerProvider, Ownership: ownership, + SpanAttributes: opts.spanAttributes, }) if err != nil { opts.logger.Error(err, "failed to create controller") diff --git a/pkg/controller/builder/options.go b/pkg/controller/builder/options.go index 1496e13..43569ba 100644 --- a/pkg/controller/builder/options.go +++ b/pkg/controller/builder/options.go @@ -12,6 +12,7 @@ import ( metricsserver "github.com/krateo-platformops/unstructured-runtime/pkg/metrics/server" "github.com/krateo-platformops/unstructured-runtime/pkg/pluralizer" "github.com/krateo-platformops/unstructured-runtime/pkg/telemetry" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "golang.org/x/time/rate" "k8s.io/client-go/util/workqueue" @@ -54,6 +55,9 @@ type options struct { // tracerProvider is controller.Options.TracerProvider; nil => the process default. tracerProvider trace.TracerProvider + // spanAttributes is controller.Options.SpanAttributes. + spanAttributes []attribute.KeyValue + // ownership is the Lease the controller must hold to reconcile; nil => none. Build fills in // its Leases client from the Configuration's rest.Config. ownership *controller.Ownership @@ -106,6 +110,15 @@ func WithGracefulShutdownTimeout(d time.Duration) func(o *options) { } } +// WithSpanAttributes sets attributes on every reconcile root span the controller starts, naming the +// controller (e.g. krateo.composition.gvr) so its traces stay attributable in a process that hosts +// several. See controller.Options.SpanAttributes. +func WithSpanAttributes(attrs ...attribute.KeyValue) func(o *options) { + return func(o *options) { + o.spanAttributes = append(o.spanAttributes, attrs...) + } +} + // WithOwnership makes the controller reconcile only while it holds the coordination.k8s.io Lease // namespace/name, as identity (the pod name), so that a second process for the same resource waits // instead of reconciling alongside. Every process that may reconcile the same resource must be given diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 32697ad..97ec6b3 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/krateo-platformops/unstructured-runtime/pkg/ratelimit" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sync" @@ -201,6 +202,12 @@ type Options struct { // krateo.io/traceparent-continued marker. Mirrors provider-runtime's WithTracerProvider. TracerProvider trace.TracerProvider + // SpanAttributes are set on every reconcile root span this controller starts, so a process that + // hosts several controllers (core-provider's cdc pool) keeps each one's traces attributable. The + // span already names the object (k8s.object.kind/name/namespace); these name the controller, e.g. + // krateo.composition.gvr. Nil: none, as before. + SpanAttributes []attribute.KeyValue + // Ownership makes Run reconcile only while it holds a coordination.k8s.io Lease, so a second // process for the same resource waits instead of reconciling alongside. Nil: no Lease. See // Ownership. @@ -255,6 +262,8 @@ type Controller struct { traces traceMemo // tracerProv is Options.TracerProvider; nil reads telemetry.TracerProvider on every pass. tracerProv trace.TracerProvider + // spanAttrs is Options.SpanAttributes, set on every reconcile root. + spanAttrs []attribute.KeyValue // runCtx is the context Run was given, cancelled at SIGTERM. A worker waiting for a local slot // waits on it, because reconcileCtx stays live through a drain. runCtx context.Context @@ -656,6 +665,7 @@ func New(sid *shortid.Shortid, opts Options) (*Controller, error) { maxRetries: opts.MaxRetries, governor: opts.RateLimitGovernor, tracerProv: opts.TracerProvider, + spanAttrs: opts.SpanAttributes, objects: newObjectGate(), gracefulShutdownTimeout: gracefulShutdownTimeout, ownership: opts.Ownership, diff --git a/pkg/controller/traces_test.go b/pkg/controller/traces_test.go index e2c4c13..a8f394d 100644 --- a/pkg/controller/traces_test.go +++ b/pkg/controller/traces_test.go @@ -15,6 +15,7 @@ import ( "github.com/krateo-platformops/unstructured-runtime/pkg/meta" "github.com/krateo-platformops/unstructured-runtime/pkg/telemetry" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" "go.opentelemetry.io/otel/trace" @@ -578,3 +579,50 @@ func TestTraceDeferredActMarksOnlyARealWrite(t *testing.T) { func TestMarkActedOutsideAPassIsANoop(t *testing.T) { MarkActed(context.Background()) } + +// Options.SpanAttributes name the controller on every reconcile root, so a process hosting several +// controllers (core-provider's cdc pool, step 3) keeps each one's traces attributable. The control +// is the same pass without the option: no such attribute. +func TestTraceRootCarriesSpanAttributes(t *testing.T) { + gvr := attribute.String("krateo.composition.gvr", "test.example.org/v1/tests") + for _, tc := range []struct { + name string + attrs []attribute.KeyValue + want bool + }{{"with SpanAttributes", []attribute.KeyValue{gvr}, true}, {"without (control)", nil, false}} { + t.Run(tc.name, func(t *testing.T) { + rec := tracetest.NewSpanRecorder() + sdk := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)) + t.Cleanup(func() { _ = sdk.Shutdown(context.Background()) }) + sid, err := shortid.New(1, shortid.DefaultABC, 2342) + require.NoError(t, err) + opts := createTestOptions() + opts.TracerProvider = sdk + opts.SpanAttributes = tc.attrs + ctrl, err := New(sid, opts) + require.NoError(t, err) + ctrl.externalClient = &actingClient{exists: true} + obj := createTestUnstructured("attrs", opts.Namespace) + _, err = opts.Client.Resource(opts.GVR).Namespace(opts.Namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + require.NoError(t, err) + require.NoError(t, ctrl.processItem(context.TODO(), ctrlevent.Event{ + EventType: ctrlevent.Observe, + ObjectRef: objectref.ObjectRef{APIVersion: "test.example.org/v1", Kind: "Test", Name: "attrs", Namespace: opts.Namespace}, + })) + var root sdktrace.ReadOnlySpan + for _, s := range rec.Ended() { + if s.Name() == tracecontext.SpanReconcile { + root = s + } + } + require.NotNil(t, root, "control: the pass started a reconcile root") + got := false + for _, kv := range root.Attributes() { + if kv.Key == gvr.Key && kv.Value.AsString() == gvr.Value.AsString() { + got = true + } + } + require.Equal(t, tc.want, got, "root carries %s", gvr.Key) + }) + } +} diff --git a/pkg/controller/worker.go b/pkg/controller/worker.go index 0b7c3fb..a18dcda 100644 --- a/pkg/controller/worker.go +++ b/pkg/controller/worker.go @@ -473,6 +473,9 @@ func (c *Controller) processItem(ctx context.Context, obj interface{}) (err erro decision := c.traces.decide(tracer != nil, traceKey, el, time.Now(), tracecontext.DefaultContinueWindow) traceObj := tracecontext.Object{Kind: el.GetKind(), Name: el.GetName(), Namespace: el.GetNamespace()} ctx, span := tracecontext.StartReconcile(ctx, tracer, traceObj, decision, time.Now()) + if len(c.spanAttrs) > 0 { + span.SetAttributes(c.spanAttrs...) // which controller (Options.SpanAttributes) + } pass := offPass if tracer != nil { pass = &tracePass{tracer: tracer, span: span, d: decision, obj: traceObj, key: traceKey, uid: el.GetUID(), memo: &c.traces, c: c, target: el} diff --git a/pkg/telemetry/metrics.go b/pkg/telemetry/metrics.go index 61102af..13c7e8a 100644 --- a/pkg/telemetry/metrics.go +++ b/pkg/telemetry/metrics.go @@ -2,6 +2,7 @@ package telemetry import ( "context" + "sync" "sync/atomic" "time" @@ -34,10 +35,33 @@ type Config struct { // Metrics exposes a small set of low-cardinality runtime metrics for // unstructured-runtime consumers. +// +// A Metrics is a VIEW: the shared instruments (meters) plus the attributes it adds to every point +// it records. Setup returns the root view, which adds none, so a process hosting one controller +// exports exactly what it always has. A process hosting several controllers (core-provider's cdc +// pool, docs/design/cdc-pooling.md step 3) gives each one its own view with With, so each +// controller's numbers stay attributable instead of merging into one process-wide stream. type Metrics struct { + *meters + + // attrs are added to every point this view records; empty for the root view. + attrs attribute.Set + // inFlight is this view's own reconciles in flight, reported per view by the gauge callback. + inFlight atomic.Int64 + // registered: this view's in-flight count is reported (the root always is; a With view once + // it first counts a reconcile, so an idle view adds no series). + registered atomic.Bool +} + +// meters holds what every view of one Metrics shares: the instruments, the process identity, the +// process-level saturation source, and the views whose in-flight counts the gauge reports. +type meters struct { log logging.Logger deploymentName string // For tracking instance identity in exported metrics + viewsMu sync.Mutex + views []*Metrics + startupSuccess metric.Int64Counter startupFailure metric.Int64Counter @@ -47,13 +71,12 @@ type Metrics struct { connectDuration metric.Float64Histogram connectFailure metric.Int64Counter - reconcileDuration metric.Float64Histogram - queueDepth metric.Int64UpDownCounter - queueWaitDuration metric.Float64Histogram - queueOldestItemAge metric.Float64Histogram - queueWorkDuration metric.Float64Histogram - reconcileInFlight metric.Int64ObservableGauge - reconcileInFlightCount atomic.Int64 + reconcileDuration metric.Float64Histogram + queueDepth metric.Int64UpDownCounter + queueWaitDuration metric.Float64Histogram + queueOldestItemAge metric.Float64Histogram + queueWorkDuration metric.Float64Histogram + reconcileInFlight metric.Int64ObservableGauge localCapacity metric.Int64ObservableGauge saturationInput metric.Float64ObservableGauge @@ -179,10 +202,12 @@ func NewWithMeter(meter metric.Meter, log logging.Logger, deploymentName string) func newMetrics(meter metric.Meter, log logging.Logger, deploymentName string) (*Metrics, error) { var err error - m := &Metrics{ + m := &Metrics{meters: &meters{ log: log, deploymentName: deploymentName, - } + }} + m.registered.Store(true) + m.views = []*Metrics{m} if m.startupSuccess, err = meter.Int64Counter("unstructured_runtime.startup.success"); err != nil { return nil, err @@ -300,7 +325,12 @@ func newMetrics(meter metric.Meter, log logging.Logger, deploymentName string) ( } _, err = meter.RegisterCallback(func(_ context.Context, observer metric.Observer) error { - observer.ObserveInt64(m.reconcileInFlight, m.reconcileInFlightCount.Load()) + m.viewsMu.Lock() + views := append([]*Metrics(nil), m.views...) + m.viewsMu.Unlock() + for _, v := range views { + observer.ObserveInt64(m.reconcileInFlight, v.inFlight.Load(), metric.WithAttributeSet(v.attrs)) + } m.observeSaturation(observer) return nil }, m.reconcileInFlight, m.localCapacity, m.saturationInput, m.saturationInputEnabled) @@ -361,7 +391,7 @@ func (m *Metrics) IncRateLimitDeferral(ctx context.Context, key string, reason r return } attrs := append(m.instanceAttrs(), attribute.String("key", key), attribute.String("cause", reason.Cause())) - m.rateLimitDeferrals.Add(ctx, 1, metric.WithAttributes(attrs...)) + m.rateLimitDeferrals.Add(ctx, 1, m.point(attrs...)) } // instanceAttrs returns metric attributes for instance identification. @@ -375,75 +405,98 @@ func (m *Metrics) instanceAttrs() []attribute.KeyValue { } } +// With returns a view of m that adds attrs to every point it records, on top of m's own. The view +// shares m's instruments, so it costs no new metric streams beyond the new attribute values. Use one +// per controller in a process that hosts several (core-provider's cdc pool), with attributes naming +// the controller. A nil m returns nil, like every other method. +// +// Saturation (local capacity and its input) is not per view: it measures the PROCESS, which every +// controller in it shares, so it keeps only the process identity. +func (m *Metrics) With(attrs ...attribute.KeyValue) *Metrics { + if m == nil { + return nil + } + merged := append(m.attrs.ToSlice(), attrs...) + return &Metrics{meters: m.meters, attrs: attribute.NewSet(merged...)} +} + +// point is the measurement option carrying this view's attributes plus extra. +func (m *Metrics) point(extra ...attribute.KeyValue) metric.MeasurementOption { + if len(extra) == 0 { + return metric.WithAttributeSet(m.attrs) + } + return metric.WithAttributes(append(m.attrs.ToSlice(), extra...)...) +} + func (m *Metrics) IncStartupSuccess(ctx context.Context) { if m == nil { return } - m.startupSuccess.Add(ctx, 1) + m.startupSuccess.Add(ctx, 1, m.point()) } func (m *Metrics) IncStartupFailure(ctx context.Context) { if m == nil { return } - m.startupFailure.Add(ctx, 1) + m.startupFailure.Add(ctx, 1, m.point()) } func (m *Metrics) RecordGetDuration(ctx context.Context, d time.Duration) { if m == nil { return } - m.getDuration.Record(ctx, d.Seconds()) + m.getDuration.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) IncGetFailure(ctx context.Context) { if m == nil { return } - m.getFailure.Add(ctx, 1) + m.getFailure.Add(ctx, 1, m.point()) } func (m *Metrics) RecordReconcileDuration(ctx context.Context, d time.Duration) { if m == nil { return } - m.reconcileDuration.Record(ctx, d.Seconds()) + m.reconcileDuration.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) RecordQueueWaitDuration(ctx context.Context, d time.Duration) { if m == nil { return } - m.queueWaitDuration.Record(ctx, d.Seconds()) + m.queueWaitDuration.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) RecordQueueWorkDuration(ctx context.Context, d time.Duration) { if m == nil { return } - m.queueWorkDuration.Record(ctx, d.Seconds()) + m.queueWorkDuration.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) recordRequeue(ctx context.Context, reason string, counter metric.Int64Counter) { if m == nil { return } - counter.Add(ctx, 1) - m.queueRequeueTotal.Add(ctx, 1, metric.WithAttributes(attribute.String("reason", reason))) + counter.Add(ctx, 1, m.point()) + m.queueRequeueTotal.Add(ctx, 1, m.point(attribute.String("reason", reason))) } func (m *Metrics) IncReconcileSuccess(ctx context.Context) { if m == nil { return } - m.reconcileSuccess.Add(ctx, 1) + m.reconcileSuccess.Add(ctx, 1, m.point()) } func (m *Metrics) IncReconcileFailure(ctx context.Context) { if m == nil { return } - m.reconcileFailure.Add(ctx, 1) + m.reconcileFailure.Add(ctx, 1, m.point()) } func (m *Metrics) IncReconcileRequeueAfter(ctx context.Context) { @@ -462,131 +515,136 @@ func (m *Metrics) AddReconcileInFlight(delta int64) { if m == nil || delta == 0 { return } - m.reconcileInFlightCount.Add(delta) + if !m.registered.Load() && m.registered.CompareAndSwap(false, true) { + m.viewsMu.Lock() + m.views = append(m.views, m) + m.viewsMu.Unlock() + } + m.inFlight.Add(delta) } func (m *Metrics) RecordObserveDuration(ctx context.Context, d time.Duration) { if m == nil { return } - m.observeDuration.Record(ctx, d.Seconds()) + m.observeDuration.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) IncObserveFailure(ctx context.Context) { if m == nil { return } - m.observeFailure.Add(ctx, 1) + m.observeFailure.Add(ctx, 1, m.point()) } func (m *Metrics) RecordCreateDuration(ctx context.Context, d time.Duration) { if m == nil { return } - m.createDuration.Record(ctx, d.Seconds()) + m.createDuration.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) IncCreateFailure(ctx context.Context) { if m == nil { return } - m.createFailure.Add(ctx, 1) + m.createFailure.Add(ctx, 1, m.point()) } func (m *Metrics) RecordUpdateDuration(ctx context.Context, d time.Duration) { if m == nil { return } - m.updateDuration.Record(ctx, d.Seconds()) + m.updateDuration.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) IncUpdateFailure(ctx context.Context) { if m == nil { return } - m.updateFailure.Add(ctx, 1) + m.updateFailure.Add(ctx, 1, m.point()) } func (m *Metrics) RecordDeleteDuration(ctx context.Context, d time.Duration) { if m == nil { return } - m.deleteDuration.Record(ctx, d.Seconds()) + m.deleteDuration.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) IncDeleteFailure(ctx context.Context) { if m == nil { return } - m.deleteFailure.Add(ctx, 1) + m.deleteFailure.Add(ctx, 1, m.point()) } func (m *Metrics) RecordFinalizerAddDuration(ctx context.Context, d time.Duration) { if m == nil { return } - m.finalizerAddDuration.Record(ctx, d.Seconds()) + m.finalizerAddDuration.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) IncFinalizerAddFailure(ctx context.Context) { if m == nil { return } - m.finalizerAddFailure.Add(ctx, 1) + m.finalizerAddFailure.Add(ctx, 1, m.point()) } func (m *Metrics) RecordFinalizerRemoveDuration(ctx context.Context, d time.Duration) { if m == nil { return } - m.finalizerRemoveDuration.Record(ctx, d.Seconds()) + m.finalizerRemoveDuration.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) IncFinalizerRemoveFailure(ctx context.Context) { if m == nil { return } - m.finalizerRemoveFailure.Add(ctx, 1) + m.finalizerRemoveFailure.Add(ctx, 1, m.point()) } func (m *Metrics) RecordStatusUpdateDuration(ctx context.Context, d time.Duration) { if m == nil { return } - m.statusUpdateDuration.Record(ctx, d.Seconds()) + m.statusUpdateDuration.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) IncStatusUpdateFailure(ctx context.Context) { if m == nil { return } - m.statusUpdateFailure.Add(ctx, 1) + m.statusUpdateFailure.Add(ctx, 1, m.point()) } func (m *Metrics) AddQueueDepth(ctx context.Context, delta int64) { if m == nil { return } - m.queueDepth.Add(ctx, delta) + m.queueDepth.Add(ctx, delta, m.point()) } func (m *Metrics) RecordQueueOldestItemAge(ctx context.Context, d time.Duration) { if m == nil { return } - m.queueOldestItemAge.Record(ctx, d.Seconds()) + m.queueOldestItemAge.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) RecordConnectDuration(ctx context.Context, d time.Duration) { if m == nil { return } - m.connectDuration.Record(ctx, d.Seconds()) + m.connectDuration.Record(ctx, d.Seconds(), m.point()) } func (m *Metrics) IncConnectFailure(ctx context.Context) { if m == nil { return } - m.connectFailure.Add(ctx, 1) + m.connectFailure.Add(ctx, 1, m.point()) } diff --git a/pkg/telemetry/views_test.go b/pkg/telemetry/views_test.go new file mode 100644 index 0000000..4b64e3c --- /dev/null +++ b/pkg/telemetry/views_test.go @@ -0,0 +1,109 @@ +package telemetry + +import ( + "context" + "testing" + + "github.com/krateo-platformops/unstructured-runtime/pkg/logging" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +const gvrKey = attribute.Key("krateo.composition.gvr") + +func collectInt64(t *testing.T, reader *sdkmetric.ManualReader) map[string][]metricdata.DataPoint[int64] { + t.Helper() + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatal(err) + } + out := map[string][]metricdata.DataPoint[int64]{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + switch d := m.Data.(type) { + case metricdata.Sum[int64]: + out[m.Name] = append(out[m.Name], d.DataPoints...) + case metricdata.Gauge[int64]: + out[m.Name] = append(out[m.Name], d.DataPoints...) + } + } + } + return out +} + +// valueBy sums the points of a metric by the value of key ("" when absent). +func valueBy(points []metricdata.DataPoint[int64], key attribute.Key) map[string]int64 { + out := map[string]int64{} + for _, p := range points { + v, _ := p.Attributes.Value(key) + out[v.AsString()] += p.Value + } + return out +} + +// core-provider docs/design/cdc-pooling.md, step 3, the proof it names: two controllers in ONE +// process produce two distinctly attributed metric streams. Each records through its own view +// (With); the root view, which a process hosting one controller uses, adds nothing, exactly as +// before. +func TestViews_TwoControllersInOneProcessAreDistinct(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + root, err := NewWithMeter(provider.Meter("test"), logging.NewNopLogger(), "") + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + a := root.With(gvrKey.String("composition.krateo.io/v1-0-0/fireworksapps")) + b := root.With(gvrKey.String("composition.krateo.io/v0-3-0/portals")) + + a.IncReconcileSuccess(ctx) + a.IncReconcileSuccess(ctx) + b.IncReconcileSuccess(ctx) + root.IncReconcileSuccess(ctx) + a.AddReconcileInFlight(2) + b.AddReconcileInFlight(1) + a.IncReconcileErrorRequeue(ctx) // a point that carries its own attribute too + + got := collectInt64(t, reader) + success := valueBy(got["unstructured_runtime.reconcile.success"], gvrKey) + if success["composition.krateo.io/v1-0-0/fireworksapps"] != 2 || success["composition.krateo.io/v0-3-0/portals"] != 1 || success[""] != 1 { + t.Fatalf("reconcile.success by %s = %v, want fireworksapps 2, portals 1, root (no attribute) 1", gvrKey, success) + } + inflight := valueBy(got["unstructured_runtime.reconcile.in_flight"], gvrKey) + if inflight["composition.krateo.io/v1-0-0/fireworksapps"] != 2 || inflight["composition.krateo.io/v0-3-0/portals"] != 1 { + t.Fatalf("in-flight by %s = %v, want each controller's own count", gvrKey, inflight) + } + requeue := got["unstructured_runtime.reconcile.queue.requeues"] + found := false + for _, p := range requeue { + gvr, _ := p.Attributes.Value(gvrKey) + reason, _ := p.Attributes.Value("reason") + if gvr.AsString() == "composition.krateo.io/v1-0-0/fireworksapps" && reason.AsString() == "error" { + found = true + } + } + if !found { + t.Fatalf("a point with its own attribute must keep the view's too; requeue points: %v", requeue) + } +} + +// An idle view adds no in-flight series; a nil Metrics stays a no-op. +func TestViews_IdleViewAddsNoSeriesAndNilIsSafe(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + root, err := NewWithMeter(provider.Meter("test"), logging.NewNopLogger(), "") + if err != nil { + t.Fatal(err) + } + _ = root.With(gvrKey.String("idle")) + for _, p := range collectInt64(t, reader)["unstructured_runtime.reconcile.in_flight"] { + if v, ok := p.Attributes.Value(gvrKey); ok && v.AsString() == "idle" { + t.Fatal("a view that never counted a reconcile must not add an in-flight series") + } + } + var nilM *Metrics + if nilM.With(gvrKey.String("x")) != nil { + t.Fatal("With on a nil Metrics must return nil") + } +}