Configurable telemetry root span behaviour

packages/cron/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @lowerdeck/cron
 
+## 1.1.0
+
+### Minor Changes
+
+- Allow to configure otel root spans
+
+### Patch Changes
+
+- Updated dependencies
+  - @lowerdeck/execution-context@1.1.0
+
 ## 1.0.7
 
 ### Patch Changes

packages/cron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lowerdeck/cron",
-  "version": "1.0.7",
+  "version": "1.1.0",
   "publishConfig": {
     "access": "public"
   },
@@ -30,7 +30,7 @@
     "build": "rm -rf ./dist && microbundle"
   },
   "dependencies": {
-    "@lowerdeck/execution-context": "^1.0.3",
+    "@lowerdeck/execution-context": "^1.1.0",
     "@lowerdeck/id": "^1.0.6",
     "@lowerdeck/queue": "^1.0.7",
     "@lowerdeck/redis": "^1.0.4",

packages/cron/src/index.ts

@@ -58,7 +58,7 @@ export let createCron = (
       let worker = new Worker(
         queue.name,
         async () => {
-          provideExecutionContext(
+          await provideExecutionContext(
             createExecutionContext({
               type: 'scheduled',
               contextId: generateCustomId('cron_'),

packages/execution-context/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @lowerdeck/execution-context
 
+## 1.1.0
+
+### Minor Changes
+
+- Allow to configure otel root spans
+
 ## 1.0.4
 
 ### Patch Changes

packages/execution-context/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lowerdeck/execution-context",
-  "version": "1.0.4",
+  "version": "1.1.0",
   "publishConfig": {
     "access": "public"
   },

packages/execution-context/src/execution-context.ts

@@ -46,6 +46,12 @@ export let createExecutionContext = (
         tracestate: carrier.tracestate,
         baggage: carrier.baggage
       };
+    } else if (input.parent?.trace) {
+      input.trace = {
+        traceparent: input.parent.trace.traceparent,
+        tracestate: input.parent.trace.tracestate,
+        baggage: input.parent.trace.baggage
+      };
     }
   }
 

packages/execution-context/src/with-execution-context.ts

@@ -5,21 +5,37 @@ import { ExecutionContext } from './execution-context';
 
 let Sentry = getSentry();
 
+let getTraceCarrier = (executionContext: ExecutionContext): Record<string, string> => {
+  let current: ExecutionContext | undefined = executionContext;
+
+  while (current) {
+    let carrier: Record<string, string> = {};
+
+    if (current.trace?.traceparent) {
+      carrier.traceparent = current.trace.traceparent;
+    }
+    if (current.trace?.tracestate) {
+      carrier.tracestate = current.trace.tracestate;
+    }
+    if (current.trace?.baggage) {
+      carrier.baggage = current.trace.baggage;
+    }
+
+    if (Object.keys(carrier).length) {
+      return carrier;
+    }
+
+    current = current.parent;
+  }
+
+  return {};
+};
+
 export let withExecutionTraceContext = async <T>(
   executionContext: ExecutionContext,
   cb: () => Promise<T>
 ): Promise<T> => {
-  let carrier: Record<string, string> = {};
-
-  if (executionContext.trace?.traceparent) {
-    carrier.traceparent = executionContext.trace.traceparent;
-  }
-  if (executionContext.trace?.tracestate) {
-    carrier.tracestate = executionContext.trace.tracestate;
-  }
-  if (executionContext.trace?.baggage) {
-    carrier.baggage = executionContext.trace.baggage;
-  }
+  let carrier = getTraceCarrier(executionContext);
 
   if (!Object.keys(carrier).length) {
     return await cb();

packages/telemetry/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @lowerdeck/telemetry
 
+## 1.1.0
+
+### Minor Changes
+
+- Allow to configure otel root spans
+
+### Patch Changes
+
+- Updated dependencies
+  - @lowerdeck/execution-context@1.1.0
+
 ## 1.0.1
 
 ### Patch Changes

packages/telemetry/README.md

@@ -16,13 +16,21 @@ pnpm add @lowerdeck/telemetry
 ```typescript
 import { initTelemetry, withExecutionContextTraceFallback } from '@lowerdeck/telemetry';
 
-initTelemetry({ serviceName: 'my-service' });
+initTelemetry({
+  serviceName: 'my-service',
+  allowRootSpans: true
+});
 
 let result = await withExecutionContextTraceFallback(async () => {
   return await doWork();
 });
 ```
 
+Notes:
+- Root spans are disabled by default.
+- Set `allowRootSpans: true` for services that should create root traces themselves.
+- `OTEL_ALLOW_ROOT_SPANS=true` also enables root spans.
+
 ## Environment Variables
 
 ### `OTEL_ENABLED`
@@ -58,6 +66,17 @@ Example:
 export OTEL_EXPORTER_OTLP_HEADERS="x-sentry-auth=sentry sentry_key=secret-key"
 ```
 
+### `OTEL_ALLOW_ROOT_SPANS`
+- Type: `'true' | 'false'`
+- Default: `'false'`
+- What it does: enables creating root spans in the service when no parent trace exists.
+
+Example:
+
+```bash
+export OTEL_ALLOW_ROOT_SPANS=true
+```
+
 ### `OTEL_SERVICE_NAME`
 - Type: string
 - Default: value passed to `initTelemetry({ serviceName })`

packages/telemetry/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lowerdeck/telemetry",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "publishConfig": {
     "access": "public"
   },
@@ -30,7 +30,7 @@
     "build": "rm -rf ./dist && microbundle"
   },
   "dependencies": {
-    "@lowerdeck/execution-context": "^1.0.4",
+    "@lowerdeck/execution-context": "^1.1.0",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/context-async-hooks": "^2.5.0",
     "@opentelemetry/core": "^2.5.0",

packages/telemetry/src/index.ts

@@ -8,13 +8,18 @@ import {
 import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
 import {
   CompositePropagator,
+  suppressTracing,
   W3CBaggagePropagator,
   W3CTraceContextPropagator
 } from '@opentelemetry/core';
 import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
 import { registerInstrumentations } from '@opentelemetry/instrumentation';
 import { resourceFromAttributes } from '@opentelemetry/resources';
-import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';
+import {
+  AlwaysOffSampler,
+  BatchSpanProcessor,
+  ParentBasedSampler
+} from '@opentelemetry/sdk-trace-base';
 import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
 import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
 import { PrismaInstrumentation } from '@prisma/instrumentation';
@@ -67,21 +72,36 @@ export let hasActiveSpan = () => !!trace.getSpan(otelContext.active());
 
 export let hasActiveTraceContext = () => !!trace.getSpanContext(otelContext.active());
 
+let withTracingSuppressed = async <T>(cb: () => Promise<T>): Promise<T> => {
+  let suppressedContext = suppressTracing(otelContext.active());
+  return await otelContext.with(suppressedContext, cb);
+};
+
 export let withExecutionContextTraceFallback = async <T>(cb: () => Promise<T>): Promise<T> => {
-  if (!isTelemetryEnabled() || hasActiveTraceContext()) {
+  if (!isTelemetryEnabled()) {
+    return await cb();
+  }
+
+  if (hasActiveTraceContext()) {
     return await cb();
   }
 
   return await withExecutionContextOptional(async executionContext => {
     if (!executionContext) {
-      return await cb();
+      return await withTracingSuppressed(cb);
     }
 
-    return await withExecutionTraceContext(executionContext, cb);
+    return await withExecutionTraceContext(executionContext, async () => {
+      if (hasActiveTraceContext()) {
+        return await cb();
+      }
+
+      return await withTracingSuppressed(cb);
+    });
   });
 };
 
-export let initTelemetry = (opts: { serviceName: string }) => {
+export let initTelemetry = (opts: { serviceName: string; allowRootSpans?: boolean }) => {
   if (initialized) return;
 
   let endpoint = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT?.trim();
@@ -93,6 +113,8 @@ export let initTelemetry = (opts: { serviceName: string }) => {
 
   let serviceName = process.env.OTEL_SERVICE_NAME?.trim() || opts.serviceName;
   let extraResourceAttributes = parseResourceAttributes(process.env.OTEL_RESOURCE_ATTRIBUTES);
+  let allowRootSpans =
+    opts.allowRootSpans === true || process.env.OTEL_ALLOW_ROOT_SPANS === 'true';
 
   let provider = new NodeTracerProvider({
     resource: resourceFromAttributes({
@@ -101,6 +123,11 @@ export let initTelemetry = (opts: { serviceName: string }) => {
         process.env.METORIAL_ENV ?? process.env.NODE_ENV ?? 'development',
       ...extraResourceAttributes
     }),
+    sampler: allowRootSpans
+      ? undefined
+      : new ParentBasedSampler({
+          root: new AlwaysOffSampler()
+        }),
     spanProcessors: [
       new BatchSpanProcessor(
         new OTLPTraceExporter({
@@ -129,5 +156,7 @@ export let initTelemetry = (opts: { serviceName: string }) => {
 
   initialized = true;
 
-  console.log(`[otel] initialized for ${serviceName} -> ${endpoint}`);
+  console.log(
+    `[otel] initialized for ${serviceName} -> ${endpoint} (allowRootSpans=${allowRootSpans})`
+  );
 };