From 411b4f57c1dde747e618a85598c477c1175591a5 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 15 Jun 2026 23:11:04 -0400 Subject: [PATCH 01/18] feat(02-02): surface UFC split serialId in eval metadata + add span-enrichment gate - Add nullable Integer serialId to UFC Split model + constructor (Moshi reflectively deserializes the serialId JSON field; no custom adapter needed) - DDEvaluator surfaces __dd_split_serial_id (when present) and __dd_do_log in OpenFeature eval metadata for APM span enrichment (JAVA-01) - Add FeatureFlaggingConfig.SPAN_ENRICHMENT_ENABLED dot-form constant mapping to DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED (distinct, off by default) - Register the env var in metadata/supported-configurations.json (version A, boolean) - Update DDEvaluatorTest Split call sites for the new constructor arg --- .../api/config/FeatureFlaggingConfig.java | 9 ++ metadata/supported-configurations.json | 8 ++ .../trace/api/openfeature/DDEvaluator.java | 8 ++ .../api/openfeature/DDEvaluatorTest.java | 84 ++++++++++--------- .../trace/api/featureflag/ufc/v1/Split.java | 10 ++- 5 files changed, 79 insertions(+), 40 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 28151f88864..e5a2edb856b 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -3,4 +3,13 @@ public class FeatureFlaggingConfig { public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; + + /** + * Opt-in gate for APM span enrichment with feature-flag evaluation metadata. The dot-form maps to + * {@code DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED} via the dot-to-underscore + + * {@code DD_} prefix normalization rule. This is DISTINCT from {@link #FLAGGING_PROVIDER_ENABLED} + * and is OFF by default — enabling the provider does not enable span enrichment. + */ + public static final String SPAN_ENRICHMENT_ENABLED = + "experimental.flagging.provider.span.enrichment.enabled"; } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index e2d171c3c3f..096b5c5b936 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1497,6 +1497,14 @@ "aliases": [] } ], + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": [] + } + ], "DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED": [ { "version": "B", diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 91c0aafdc7a..aeafbac6ba7 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -392,6 +392,14 @@ private static ProviderEvaluation resolveVariant( .addString("flagKey", flag.key) .addString("variationType", flag.variationType.name()) .addString("allocationKey", allocation.key); + // Surface the UFC split's serial id and the allocation's doLog flag for APM span enrichment + // (JAVA-01). __dd_split_serial_id is omitted when the split carries no serial id; __dd_do_log + // is always present so the span-enrichment hook can decide whether to record the subject. + if (split.serialId != null) { + metadataBuilder.addString("__dd_split_serial_id", split.serialId.toString()); + } + metadataBuilder.addString( + "__dd_do_log", String.valueOf(allocation.doLog != null && allocation.doLog)); final ProviderEvaluation result = ProviderEvaluation.builder() .value(mappedValue) diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java index bb86c409bad..db2d791d3bd 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DDEvaluatorTest.java @@ -641,7 +641,7 @@ private ServerConfiguration createTestConfiguration() { private Flag createSimpleFlag(String key, ValueType type, Object value, String variantKey) { final Map variants = new HashMap<>(); variants.put(variantKey, new Variant(variantKey, value)); - final List splits = singletonList(new Split(emptyList(), variantKey, null)); + final List splits = singletonList(new Split(emptyList(), variantKey, null, null)); final List allocations = singletonList(new Allocation("alloc1", null, null, null, splits, false)); return new Flag(key, true, type, variants, allocations); @@ -657,12 +657,12 @@ private Flag createRuleBasedFlag() { singletonList( new ConditionConfiguration(ConditionOperator.MATCHES, "email", "@company\\.com$")); final List premiumRules = singletonList(new Rule(premiumConditions)); - final List premiumSplits = singletonList(new Split(emptyList(), "premium", null)); + final List premiumSplits = singletonList(new Split(emptyList(), "premium", null, null)); final Allocation premiumAllocation = new Allocation("premium-alloc", premiumRules, null, null, premiumSplits, false); // Fallback allocation for basic - final List basicSplits = singletonList(new Split(emptyList(), "basic", null)); + final List basicSplits = singletonList(new Split(emptyList(), "basic", null, null)); final Allocation basicAllocation = new Allocation("basic-alloc", null, null, null, basicSplits, false); @@ -680,12 +680,12 @@ private Flag createNumericRuleFlag() { final List vipConditions = singletonList(new ConditionConfiguration(ConditionOperator.GTE, "score", 800)); final List vipRules = singletonList(new Rule(vipConditions)); - final List vipSplits = singletonList(new Split(emptyList(), "vip", null)); + final List vipSplits = singletonList(new Split(emptyList(), "vip", null, null)); final Allocation vipAllocation = new Allocation("vip-alloc", vipRules, null, null, vipSplits, false); // Fallback - final List regularSplits = singletonList(new Split(emptyList(), "regular", null)); + final List regularSplits = singletonList(new Split(emptyList(), "regular", null, null)); final Allocation regularAllocation = new Allocation("regular-alloc", null, null, null, regularSplits, false); @@ -706,12 +706,12 @@ private Flag createNullCheckFlag() { final List noBetaConditions = singletonList(new ConditionConfiguration(ConditionOperator.IS_NULL, "beta_feature", true)); final List noBetaRules = singletonList(new Rule(noBetaConditions)); - final List noBetaSplits = singletonList(new Split(emptyList(), "no-beta", null)); + final List noBetaSplits = singletonList(new Split(emptyList(), "no-beta", null, null)); final Allocation noBetaAllocation = new Allocation("no-beta-alloc", noBetaRules, null, null, noBetaSplits, false); // Fallback - final List hasBetaSplits = singletonList(new Split(emptyList(), "has-beta", null)); + final List hasBetaSplits = singletonList(new Split(emptyList(), "has-beta", null, null)); final Allocation hasBetaAllocation = new Allocation("has-beta-alloc", null, null, null, hasBetaSplits, false); @@ -734,12 +734,13 @@ private Flag createOneOfRuleFlag() { singletonList( new ConditionConfiguration(ConditionOperator.ONE_OF, "region", allowedRegions)); final List regionalRules = singletonList(new Rule(regionalConditions)); - final List regionalSplits = singletonList(new Split(emptyList(), "regional", null)); + final List regionalSplits = + singletonList(new Split(emptyList(), "regional", null, null)); final Allocation regionalAllocation = new Allocation("regional-alloc", regionalRules, null, null, regionalSplits, false); // Fallback - final List globalSplits = singletonList(new Split(emptyList(), "global", null)); + final List globalSplits = singletonList(new Split(emptyList(), "global", null, null)); final Allocation globalAllocation = new Allocation("global-alloc", null, null, null, globalSplits, false); @@ -755,7 +756,7 @@ private Flag createTimeBasedFlag() { final Map variants = new HashMap<>(); variants.put("time-limited", new Variant("time-limited", "time-limited")); - final List splits = singletonList(new Split(emptyList(), "time-limited", null)); + final List splits = singletonList(new Split(emptyList(), "time-limited", null, null)); // Allocation that ended in 2022 (should be inactive) final List allocations = @@ -779,7 +780,7 @@ private Flag createShardBasedFlag() { final List ranges = singletonList(new ShardRange(0, 50)); // 0-49 out of 100 final List shards = singletonList(new Shard("test-salt", ranges, 100)); - final List splits = singletonList(new Split(shards, "shard-variant", null)); + final List splits = singletonList(new Split(shards, "shard-variant", null, null)); final List allocations = singletonList(new Allocation("shard-alloc", null, null, null, splits, false)); @@ -792,7 +793,7 @@ private Flag createBrokenFlag() { final Map variants = new HashMap<>(); variants.put("existing", new Variant("existing", "value")); - final List splits = singletonList(new Split(emptyList(), "missing-variant", null)); + final List splits = singletonList(new Split(emptyList(), "missing-variant", null, null)); final List allocations = singletonList(new Allocation("alloc1", null, null, null, splits, false)); @@ -814,7 +815,7 @@ private Flag createComparisonFlag( final List conditions = singletonList(new ConditionConfiguration(operator, attribute, threshold)); final List rules = singletonList(new Rule(conditions)); - final List splits = singletonList(new Split(emptyList(), variantKey, null)); + final List splits = singletonList(new Split(emptyList(), variantKey, null, null)); final Allocation allocation = new Allocation(allocKey, rules, null, null, splits, false); return new Flag(flagKey, true, ValueType.STRING, variants, singletonList(allocation)); @@ -849,7 +850,7 @@ private Flag createNotOperatorFlag( final List conditions = singletonList(new ConditionConfiguration(operator, attribute, value)); final List rules = singletonList(new Rule(conditions)); - final List splits = singletonList(new Split(emptyList(), variantKey, null)); + final List splits = singletonList(new Split(emptyList(), variantKey, null, null)); final Allocation allocation = new Allocation(allocKey, rules, null, null, splits, false); return new Flag(flagKey, true, ValueType.STRING, variants, singletonList(allocation)); @@ -886,7 +887,7 @@ private Flag createDoubleEqualsFlag() { final List piConditions = singletonList(new ConditionConfiguration(ConditionOperator.MATCHES, "rate", "3.14159")); final List piRules = singletonList(new Rule(piConditions)); - final List piSplits = singletonList(new Split(emptyList(), "pi", null)); + final List piSplits = singletonList(new Split(emptyList(), "pi", null, null)); final Allocation piAllocation = new Allocation("pi-alloc", piRules, null, null, piSplits, false); @@ -903,7 +904,7 @@ private Flag createNestedAttributeFlag() { singletonList( new ConditionConfiguration(ConditionOperator.MATCHES, "user.profile.level", "premium")); final List premiumRules = singletonList(new Rule(premiumConditions)); - final List premiumSplits = singletonList(new Split(emptyList(), "premium", null)); + final List premiumSplits = singletonList(new Split(emptyList(), "premium", null, null)); final Allocation premiumAllocation = new Allocation("premium-nested-alloc", premiumRules, null, null, premiumSplits, false); @@ -915,7 +916,7 @@ private Flag createExposureFlag() { final Map variants = new HashMap<>(); variants.put("tracked", new Variant("tracked", "tracked-value")); - final List splits = singletonList(new Split(emptyList(), "tracked", null)); + final List splits = singletonList(new Split(emptyList(), "tracked", null, null)); // Create allocation with doLog=true to trigger exposure logging final List allocations = singletonList(new Allocation("exposure-alloc", null, null, null, splits, true)); @@ -931,7 +932,7 @@ private Flag createDoubleComparisonFlag() { final List exactConditions = singletonList(new ConditionConfiguration(ConditionOperator.LTE, "score", 3.14159)); final List exactRules = singletonList(new Rule(exactConditions)); - final List exactSplits = singletonList(new Split(emptyList(), "exact", null)); + final List exactSplits = singletonList(new Split(emptyList(), "exact", null, null)); final Allocation exactAllocation = new Allocation("exact-alloc", exactRules, null, null, exactSplits, false); @@ -947,7 +948,7 @@ private Flag createExposureLoggingFlag() { final List loggedConditions = singletonList(new ConditionConfiguration(ConditionOperator.MATCHES, "feature", "premium")); final List loggedRules = singletonList(new Rule(loggedConditions)); - final List loggedSplits = singletonList(new Split(emptyList(), "logged", null)); + final List loggedSplits = singletonList(new Split(emptyList(), "logged", null, null)); // Create allocation with doLog=true to trigger exposure logging and allocationKey method final Allocation loggedAllocation = new Allocation("logged-alloc", loggedRules, null, null, loggedSplits, true); @@ -966,7 +967,8 @@ private Flag createNumericOneOfFlag() { final List numericConditions = singletonList(new ConditionConfiguration(ConditionOperator.ONE_OF, "score", numericValues)); final List numericRules = singletonList(new Rule(numericConditions)); - final List numericSplits = singletonList(new Split(emptyList(), "numeric-match", null)); + final List numericSplits = + singletonList(new Split(emptyList(), "numeric-match", null, null)); final Allocation numericAllocation = new Allocation("numeric-alloc", numericRules, null, null, numericSplits, false); @@ -985,7 +987,8 @@ private Flag createNumericNotOneOfFlag() { singletonList( new ConditionConfiguration(ConditionOperator.NOT_ONE_OF, "score", excludedValues)); final List excludedRules = singletonList(new Rule(excludedConditions)); - final List excludedSplits = singletonList(new Split(emptyList(), "excluded", null)); + final List excludedSplits = + singletonList(new Split(emptyList(), "excluded", null, null)); final Allocation excludedAllocation = new Allocation("excluded-alloc", excludedRules, null, null, excludedSplits, false); @@ -1005,7 +1008,7 @@ private Flag createIsNullFalseFlag() { final List notNullConditions = singletonList(new ConditionConfiguration(ConditionOperator.IS_NULL, "attr", false)); final List notNullRules = singletonList(new Rule(notNullConditions)); - final List notNullSplits = singletonList(new Split(emptyList(), "not-null", null)); + final List notNullSplits = singletonList(new Split(emptyList(), "not-null", null, null)); final Allocation notNullAllocation = new Allocation("not-null-alloc", notNullRules, null, null, notNullSplits, false); @@ -1022,7 +1025,7 @@ private Flag createIsNullNonBooleanFlag() { singletonList( new ConditionConfiguration(ConditionOperator.IS_NULL, "missing_attr", "string")); final List nullRules = singletonList(new Rule(nullConditions)); - final List nullSplits = singletonList(new Split(emptyList(), "null-match", null)); + final List nullSplits = singletonList(new Split(emptyList(), "null-match", null, null)); final Allocation nullAllocation = new Allocation("null-alloc", nullRules, null, null, nullSplits, false); @@ -1042,7 +1045,8 @@ private Flag createNullAttributeFlag() { final List nullAttrConditions = singletonList(new ConditionConfiguration(ConditionOperator.MATCHES, null, "test")); final List nullAttrRules = singletonList(new Rule(nullAttrConditions)); - final List nullAttrSplits = singletonList(new Split(emptyList(), "fallback", null)); + final List nullAttrSplits = + singletonList(new Split(emptyList(), "fallback", null, null)); final Allocation nullAttrAllocation = new Allocation("null-attr-alloc", nullAttrRules, null, null, nullAttrSplits, false); @@ -1059,7 +1063,8 @@ private Flag createNotMatchesPositiveFlag() { singletonList( new ConditionConfiguration(ConditionOperator.NOT_MATCHES, "email", "@company\\.com")); final List externalRules = singletonList(new Rule(externalConditions)); - final List externalSplits = singletonList(new Split(emptyList(), "external", null)); + final List externalSplits = + singletonList(new Split(emptyList(), "external", null, null)); final Allocation externalAllocation = new Allocation("external-alloc", externalRules, null, null, externalSplits, false); @@ -1081,7 +1086,7 @@ private Flag createNotOneOfPositiveFlag() { singletonList( new ConditionConfiguration(ConditionOperator.NOT_ONE_OF, "region", excludedRegions)); final List otherRules = singletonList(new Rule(otherConditions)); - final List otherSplits = singletonList(new Split(emptyList(), "other", null)); + final List otherSplits = singletonList(new Split(emptyList(), "other", null, null)); final Allocation otherAllocation = new Allocation("other-alloc", otherRules, null, null, otherSplits, false); @@ -1101,7 +1106,8 @@ private Flag createFalseNumericComparisonsFlag() { final List highScoreConditions = singletonList(new ConditionConfiguration(ConditionOperator.GTE, "score", 800)); final List highScoreRules = singletonList(new Rule(highScoreConditions)); - final List highScoreSplits = singletonList(new Split(emptyList(), "high-score", null)); + final List highScoreSplits = + singletonList(new Split(emptyList(), "high-score", null, null)); final Allocation highScoreAllocation = new Allocation("high-score-alloc", highScoreRules, null, null, highScoreSplits, false); @@ -1131,7 +1137,7 @@ private Flag createEmptyConditionsFlag() { // Rule with empty conditions list - this will be skipped, causing allocation to not match final Rule emptyConditionsRule = new Rule(emptyList()); - final List splits = singletonList(new Split(emptyList(), "default", null)); + final List splits = singletonList(new Split(emptyList(), "default", null, null)); final Allocation allocation = new Allocation( "empty-conditions-alloc", @@ -1153,7 +1159,7 @@ private Flag createShardMatchingFlag() { final List ranges = singletonList(new ShardRange(0, 100)); // Full range to ensure match final List shards = singletonList(new Shard("test-salt", ranges, 100)); - final List splits = singletonList(new Split(shards, "matched", null)); + final List splits = singletonList(new Split(shards, "matched", null, null)); final Allocation allocation = new Allocation("shard-matching-alloc", null, null, null, splits, false); @@ -1165,7 +1171,7 @@ private Flag createFutureAllocationFlag() { final Map variants = new HashMap<>(); variants.put("future", new Variant("future", "future-value")); - final List splits = singletonList(new Split(emptyList(), "future", null)); + final List splits = singletonList(new Split(emptyList(), "future", null, null)); // Allocation that starts in the future (2050) final Allocation allocation = @@ -1185,7 +1191,7 @@ private Flag createIdAttributeFlag() { singletonList( new ConditionConfiguration(ConditionOperator.MATCHES, "id", "user-special-id")); final List rules = singletonList(new Rule(conditions)); - final List splits = singletonList(new Split(emptyList(), "id-match", null)); + final List splits = singletonList(new Split(emptyList(), "id-match", null, null)); final Allocation allocation = new Allocation("id-attr-alloc", rules, null, null, splits, false); return new Flag( @@ -1200,7 +1206,7 @@ private Flag createNonIterableConditionFlag() { final List conditions = singletonList(new ConditionConfiguration(ConditionOperator.ONE_OF, "attr", "single-value")); final List rules = singletonList(new Rule(conditions)); - final List splits = singletonList(new Split(emptyList(), "no-match", null)); + final List splits = singletonList(new Split(emptyList(), "no-match", null, null)); final Allocation allocation = new Allocation("non-iterable-alloc", rules, null, null, splits, false); @@ -1250,7 +1256,7 @@ private Flag createNotMatchesFalseFlag() { singletonList( new ConditionConfiguration(ConditionOperator.NOT_MATCHES, "email", "@company\\.com")); final List rules = singletonList(new Rule(conditions)); - final List splits = singletonList(new Split(emptyList(), "internal", null)); + final List splits = singletonList(new Split(emptyList(), "internal", null, null)); final Allocation allocation = new Allocation("not-matches-false-alloc", rules, null, null, splits, false); @@ -1269,7 +1275,7 @@ private Flag createNotOneOfFalseFlag() { singletonList( new ConditionConfiguration(ConditionOperator.NOT_ONE_OF, "region", excludedRegions)); final List rules = singletonList(new Rule(conditions)); - final List splits = singletonList(new Split(emptyList(), "excluded", null)); + final List splits = singletonList(new Split(emptyList(), "excluded", null, null)); final Allocation allocation = new Allocation("not-one-of-false-alloc", rules, null, null, splits, false); @@ -1285,7 +1291,7 @@ private Flag createNullContextValuesFlag() { final List conditions = singletonList(new ConditionConfiguration(ConditionOperator.IS_NULL, "nullAttr", true)); final List rules = singletonList(new Rule(conditions)); - final List splits = singletonList(new Split(emptyList(), "null-variant", null)); + final List splits = singletonList(new Split(emptyList(), "null-variant", null, null)); final Allocation allocation = new Allocation("null-context-alloc", rules, null, null, splits, false); @@ -1303,12 +1309,12 @@ private Flag createCountryRuleFlag() { final List usConditions = singletonList(new ConditionConfiguration(ConditionOperator.ONE_OF, "country", usCountries)); final List usRules = singletonList(new Rule(usConditions)); - final List usSplits = singletonList(new Split(emptyList(), "us", null)); + final List usSplits = singletonList(new Split(emptyList(), "us", null, null)); final Allocation usAllocation = new Allocation("us-alloc", usRules, null, null, usSplits, false); // Fallback allocation (no rules, no shards) - final List globalSplits = singletonList(new Split(emptyList(), "global", null)); + final List globalSplits = singletonList(new Split(emptyList(), "global", null, null)); final Allocation globalAllocation = new Allocation("global-alloc", null, null, null, globalSplits, false); @@ -1328,7 +1334,7 @@ private Flag createInvalidRegexFlag() { final List conditions = singletonList(new ConditionConfiguration(ConditionOperator.MATCHES, "email", "[invalid")); final List rules = singletonList(new Rule(conditions)); - final List splits = singletonList(new Split(emptyList(), "matched", null)); + final List splits = singletonList(new Split(emptyList(), "matched", null, null)); final Allocation allocation = new Allocation("invalid-regex-alloc", rules, null, null, splits, false); @@ -1345,7 +1351,7 @@ private Flag createInvalidRegexNotMatchesFlag() { singletonList( new ConditionConfiguration(ConditionOperator.NOT_MATCHES, "email", "[invalid")); final List rules = singletonList(new Rule(conditions)); - final List splits = singletonList(new Split(emptyList(), "excluded", null)); + final List splits = singletonList(new Split(emptyList(), "excluded", null, null)); final Allocation allocation = new Allocation("invalid-regex-not-matches-alloc", rules, null, null, splits, false); diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/Split.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/Split.java index 1782032278d..73abb1ef1ec 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/Split.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/Split.java @@ -7,11 +7,19 @@ public class Split { public final List shards; public final String variationKey; public final Map extraLogging; + // Nullable Integer (not primitive int): the serialId is absent in some UFC shapes. Populated by + // Moshi reflective deserialization from the UFC "serialId" JSON field. Surfaced as + // __dd_split_serial_id in eval metadata for APM span enrichment (JAVA-01). + public final Integer serialId; public Split( - final List shards, final String variationKey, final Map extraLogging) { + final List shards, + final String variationKey, + final Map extraLogging, + final Integer serialId) { this.shards = shards; this.variationKey = variationKey; this.extraLogging = extraLogging; + this.serialId = serialId; } } From c076e237a2d94c6d53883de56bb6d6bb5859a4bf Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 15 Jun 2026 23:24:16 -0400 Subject: [PATCH 02/18] feat(02-02): ULeb128 codec + accumulator + capture hook + write interceptor (gate-gated) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ULeb128Encoder: ULEB128 delta-varint + base64 codec ported verbatim from the frozen Node reference (dd-trace-js#8343); golden vector {100,108,128,130} -> ZAgUAg==, SHA-256 hex targeting-key hashing; JDK stdlib only (java.util.Base64, MessageDigest) - SpanEnrichmentAccumulator: per-trace state keyed by trace id in a shared ConcurrentHashMap; frozen limits (200/10/20/5/64), UTF-8-safe truncation, object default -> JSON (Pattern F tag shapes: ffe_flags_enc bare base64, ffe_subjects_enc / ffe_runtime_defaults JSON objects) - SpanEnrichmentHook: OpenFeature finally hook (capture) resolving the active local root via AgentTracer and applying the Node branch (serialId -> addSerialId/addSubject; missing variant -> addDefault); error-isolated - SpanEnrichmentInterceptor: TraceInterceptor (write) flushing ffe_* onto the local root onTraceComplete with unique priority 4, then clearing state; disable() for provider-close cleanup - Provider: gate-gated construction via ConfigHelper.env (DG-005 — nothing built/ registered when off), hook added to getProviderHooks, interceptor registered via GlobalTracer, shutdown() disables interceptor + drains state - feature-flagging-api: add compileOnly/testImplementation :internal-api for the tracer/interceptor types (runtime-provided by the agent) --- .../feature-flagging-api/build.gradle.kts | 7 + .../trace/api/openfeature/Provider.java | 82 ++++- .../SpanEnrichmentAccumulator.java | 291 ++++++++++++++++++ .../api/openfeature/SpanEnrichmentHook.java | 134 ++++++++ .../SpanEnrichmentInterceptor.java | 96 ++++++ .../trace/api/openfeature/ULeb128Encoder.java | 119 +++++++ 6 files changed, 726 insertions(+), 3 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java create mode 100644 products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java create mode 100644 products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java create mode 100644 products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java diff --git a/products/feature-flagging/feature-flagging-api/build.gradle.kts b/products/feature-flagging/feature-flagging-api/build.gradle.kts index 1c368d51b51..e3edd19cadc 100644 --- a/products/feature-flagging/feature-flagging-api/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-api/build.gradle.kts @@ -45,11 +45,18 @@ dependencies { compileOnly(project(":products:feature-flagging:feature-flagging-bootstrap")) compileOnly(project(":utils:config-utils")) + // Span enrichment (JAVA-01): TraceInterceptor / GlobalTracer / AgentTracer / AgentSpan for the + // write tier + active-root-span lookup. compileOnly because the agent runtime + // (feature-flagging-agent depends on :internal-api) provides these classes; the published + // dd-openfeature jar must not bundle the tracer. + compileOnly(project(":internal-api")) compileOnly("io.opentelemetry:opentelemetry-api:1.47.0") compileOnly("io.opentelemetry:opentelemetry-sdk-metrics:1.47.0") compileOnly("io.opentelemetry:opentelemetry-exporter-otlp:1.47.0") testImplementation(project(":products:feature-flagging:feature-flagging-bootstrap")) + testImplementation(project(":internal-api")) + testImplementation(project(":utils:config-utils")) testImplementation("io.opentelemetry:opentelemetry-api:1.47.0") testImplementation("io.opentelemetry:opentelemetry-sdk-metrics:1.47.0") testImplementation("io.opentelemetry:opentelemetry-exporter-otlp:1.47.0") diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index c492ef49c69..71377c8a39b 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -2,6 +2,8 @@ import static java.util.concurrent.TimeUnit.SECONDS; +import datadog.trace.api.GlobalTracer; +import datadog.trace.config.inversion.ConfigHelper; import de.thetaphi.forbiddenapis.SuppressForbidden; import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.EvaluationContext; @@ -16,6 +18,7 @@ import dev.openfeature.sdk.exceptions.OpenFeatureError; import dev.openfeature.sdk.exceptions.ProviderNotReadyError; import java.lang.reflect.Constructor; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -28,6 +31,15 @@ public class Provider extends EventProvider implements Metadata { private static final Logger log = LoggerFactory.getLogger(Provider.class); static final String METADATA = "datadog-openfeature-provider"; private static final String EVALUATOR_IMPL = "datadog.trace.api.openfeature.DDEvaluator"; + + /** + * Environment variable form of {@link + * datadog.trace.api.config.FeatureFlaggingConfig#SPAN_ENRICHMENT_ENABLED}. Distinct from the + * provider-enabled gate; OFF by default (experimental opt-in). + */ + static final String SPAN_ENRICHMENT_ENABLED_ENV = + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED"; + private static final Options DEFAULT_OPTIONS = new Options().initTimeout(30, SECONDS); private volatile Evaluator evaluator; private final Options options; @@ -35,6 +47,9 @@ public class Provider extends EventProvider implements Metadata { new AtomicReference<>(InitializationState.NOT_STARTED); private final FlagEvalMetrics flagEvalMetrics; private final FlagEvalHook flagEvalHook; + // Span enrichment (JAVA-01): both are null unless the gate is on (DG-005 — no idle overhead). + private final SpanEnrichmentHook spanEnrichmentHook; + private final SpanEnrichmentInterceptor spanEnrichmentInterceptor; public Provider() { this(DEFAULT_OPTIONS, null); @@ -45,6 +60,17 @@ public Provider(final Options options) { } Provider(final Options options, final Evaluator evaluator) { + this(options, evaluator, null); + } + + /** + * @param spanEnrichmentEnabledOverride when non-null, forces the span-enrichment gate (test + * seam); when null, the gate is read from {@link #SPAN_ENRICHMENT_ENABLED_ENV}. + */ + Provider( + final Options options, + final Evaluator evaluator, + final Boolean spanEnrichmentEnabledOverride) { this.options = options; this.evaluator = evaluator; FlagEvalMetrics metrics = null; @@ -59,6 +85,38 @@ public Provider(final Options options) { } this.flagEvalMetrics = metrics; this.flagEvalHook = hook; + + // Gate-gated span enrichment: construct + register ONLY when the gate is on. When off, nothing + // is constructed and no interceptor/state exists (DG-005 zero-idle-overhead negative control). + final boolean spanEnrichmentEnabled = + spanEnrichmentEnabledOverride != null + ? spanEnrichmentEnabledOverride + : isSpanEnrichmentEnabled(); + SpanEnrichmentHook seHook = null; + SpanEnrichmentInterceptor seInterceptor = null; + if (spanEnrichmentEnabled) { + try { + seHook = new SpanEnrichmentHook(); + seInterceptor = new SpanEnrichmentInterceptor(); + GlobalTracer.get().addTraceInterceptor(seInterceptor); + } catch (LinkageError | Exception e) { + // Tracer classes absent (e.g. API-only classpath): degrade to no span enrichment. + log.warn("Span enrichment unavailable — tracer classes not on classpath", e); + seHook = null; + seInterceptor = null; + } + } + this.spanEnrichmentHook = seHook; + this.spanEnrichmentInterceptor = seInterceptor; + } + + private static boolean isSpanEnrichmentEnabled() { + try { + final String value = ConfigHelper.env(SPAN_ENRICHMENT_ENABLED_ENV); + return "true".equalsIgnoreCase(value) || "1".equals(value); + } catch (final Throwable t) { + return false; // never let config reading break provider construction + } } @Override @@ -168,10 +226,14 @@ private Evaluator buildEvaluator() throws Exception { @Override public List getProviderHooks() { - if (flagEvalHook == null) { - return Collections.emptyList(); + final List hooks = new ArrayList<>(2); + if (flagEvalHook != null) { + hooks.add(flagEvalHook); } - return Collections.singletonList(flagEvalHook); + if (spanEnrichmentHook != null) { + hooks.add(spanEnrichmentHook); + } + return hooks.isEmpty() ? Collections.emptyList() : hooks; } @Override @@ -179,11 +241,25 @@ public void shutdown() { if (flagEvalMetrics != null) { flagEvalMetrics.shutdown(); } + // Provider-close cleanup for span enrichment (Pitfall 3): the tracer has no interceptor-removal + // API, so disable the interceptor (it then no-ops + drains residual state) and drop the hook. + if (spanEnrichmentInterceptor != null) { + spanEnrichmentInterceptor.disable(); + } if (evaluator != null) { evaluator.shutdown(); } } + // Visible for tests: expose whether span enrichment is wired (gate-on) without leaking the impls. + SpanEnrichmentHook spanEnrichmentHook() { + return spanEnrichmentHook; + } + + SpanEnrichmentInterceptor spanEnrichmentInterceptor() { + return spanEnrichmentInterceptor; + } + @Override public Metadata getMetadata() { return this; diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java new file mode 100644 index 00000000000..655594c153b --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java @@ -0,0 +1,291 @@ +package datadog.trace.api.openfeature; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Per-local-root-span accumulator for APM feature-flag span enrichment (JAVA-01). + * + *

Holds the serial ids, hashed subjects, and runtime defaults captured during flag evaluation + * for a single local trace fragment. The limits, dedupe semantics, truncation, and output tag + * shapes are FROZEN against the Node reference ({@code dd-trace-js#8343}) — see {@link + * ULeb128Encoder}. + * + *

Instances are created lazily and stored in {@link #STATES}, keyed by the local-root span's + * trace id. The capture hook ({@link SpanEnrichmentHook}) writes; the write interceptor ({@link + * SpanEnrichmentInterceptor}) reads and clears. When the span-enrichment gate is off, nothing + * touches {@link #STATES}, so there is no idle per-span overhead (DG-005). + * + *

Output tag shapes (Pattern F): + * + *

    + *
  • {@code ffe_flags_enc} — a bare base64 string (delta-varint of the serial ids) + *
  • {@code ffe_subjects_enc} — a JSON object string {@code {"": "", ...}} + *
  • {@code ffe_runtime_defaults} — a JSON object string {@code {"": "", ...}} + *
+ */ +final class SpanEnrichmentAccumulator { + + static final int MAX_SERIAL_IDS = 200; + static final int MAX_SUBJECTS = 10; + static final int MAX_EXPERIMENTS_PER_SUBJECT = 20; + static final int MAX_DEFAULTS = 5; + static final int MAX_DEFAULT_VALUE_LENGTH = 64; + + static final String TAG_FLAGS_ENC = "ffe_flags_enc"; + static final String TAG_SUBJECTS_ENC = "ffe_subjects_enc"; + static final String TAG_RUNTIME_DEFAULTS = "ffe_runtime_defaults"; + + /** + * Shared per-trace state, keyed by the local-root span's trace id ({@code DDTraceId.toLong()}). + * Capture (hook) puts; write (interceptor) reads + removes. Only ever populated when the gate is + * on (DG-005). + */ + static final ConcurrentHashMap STATES = + new ConcurrentHashMap<>(); + + // dedupe is structural (a Set); sorted for deterministic encoding. + private final TreeSet serialIds = new TreeSet<>(); + // sha256hex(targetingKey) -> serial ids. LinkedHashMap for stable iteration order. + private final Map> subjects = new LinkedHashMap<>(); + // flagKey -> value string (first-wins, truncated to MAX_DEFAULT_VALUE_LENGTH). + private final Map defaults = new LinkedHashMap<>(); + + /** Adds a serial id, dropping silently once {@link #MAX_SERIAL_IDS} is reached. */ + synchronized void addSerialId(final int id) { + if (serialIds.size() >= MAX_SERIAL_IDS && !serialIds.contains(id)) { + return; + } + serialIds.add(id); + } + + /** + * Records that the given targeting key was exposed to the experiment identified by {@code id}. + * The targeting key is SHA-256-hashed before storage. Enforces both the subject cap ({@link + * #MAX_SUBJECTS}) and the per-subject experiment cap ({@link #MAX_EXPERIMENTS_PER_SUBJECT}). + */ + synchronized void addSubject(final String targetingKey, final int id) { + if (targetingKey == null) { + return; + } + final String hashed = ULeb128Encoder.hashTargetingKey(targetingKey); + final TreeSet existing = subjects.get(hashed); + if (existing != null) { + if (existing.size() >= MAX_EXPERIMENTS_PER_SUBJECT && !existing.contains(id)) { + return; + } + existing.add(id); + return; + } + if (subjects.size() >= MAX_SUBJECTS) { + return; + } + final TreeSet ids = new TreeSet<>(); + ids.add(id); + subjects.put(hashed, ids); + } + + /** + * Records a runtime-default value for {@code flagKey} (first-wins). Object values are serialized + * to JSON (NOT {@code toString()}); the result is truncated to {@link #MAX_DEFAULT_VALUE_LENGTH}. + */ + synchronized void addDefault(final String flagKey, final Object value) { + if (flagKey == null) { + return; + } + if (defaults.containsKey(flagKey)) { + return; // first-wins + } + if (defaults.size() >= MAX_DEFAULTS) { + return; + } + String valueStr = stringifyDefault(value); + if (valueStr.length() > MAX_DEFAULT_VALUE_LENGTH) { + valueStr = utf8SafeTruncate(valueStr, MAX_DEFAULT_VALUE_LENGTH); + } + defaults.put(flagKey, valueStr); + } + + /** + * @return true when there is at least one serial id or runtime default to write. Subjects are not + * checked because a subject is never recorded without its serial id. + */ + synchronized boolean hasData() { + return !serialIds.isEmpty() || !defaults.isEmpty(); + } + + /** + * Builds the {@code ffe_*} span tags from the accumulated state. Empty groups are omitted. + * + * @return a map of tag name to tag value (a subset of {@code ffe_flags_enc}, {@code + * ffe_subjects_enc}, {@code ffe_runtime_defaults}) + */ + synchronized Map toSpanTags() { + final Map tags = new LinkedHashMap<>(); + if (!serialIds.isEmpty()) { + final String encoded = ULeb128Encoder.encodeDeltaVarint(serialIds); + if (!encoded.isEmpty()) { + tags.put(TAG_FLAGS_ENC, encoded); + } + } + if (!subjects.isEmpty()) { + final Map encodedSubjects = new LinkedHashMap<>(); + for (final Map.Entry> entry : subjects.entrySet()) { + encodedSubjects.put(entry.getKey(), ULeb128Encoder.encodeDeltaVarint(entry.getValue())); + } + tags.put(TAG_SUBJECTS_ENC, toJsonObject(encodedSubjects)); + } + if (!defaults.isEmpty()) { + tags.put(TAG_RUNTIME_DEFAULTS, toJsonObject(defaults)); + } + return tags; + } + + // ---- helpers (visible for tests) ---- + + /** + * Mirrors the Node {@code (typeof object && !null) ? JSON.stringify(value) : String(value)} rule: + * structured values (maps, lists) are JSON-stringified, scalars use their string form, null + * becomes the JSON literal {@code null}. + */ + static String stringifyDefault(final Object value) { + if (value == null) { + return "null"; + } + if (value instanceof Map || value instanceof Iterable || value.getClass().isArray()) { + return toJsonValue(value); + } + if (value instanceof CharSequence || value instanceof Character) { + return value.toString(); + } + // Numbers / booleans — their string form matches JSON for these scalar cases. + return String.valueOf(value); + } + + /** UTF-8-safe truncation: never split a surrogate pair at the {@code maxChars} boundary. */ + static String utf8SafeTruncate(final String value, final int maxChars) { + if (value.length() <= maxChars) { + return value; + } + int end = maxChars; + if (Character.isHighSurrogate(value.charAt(end - 1))) { + end--; // drop the dangling high surrogate rather than emit a broken pair + } + return value.substring(0, end); + } + + /** Serializes a String->String map to a compact JSON object string (keys + values escaped). */ + static String toJsonObject(final Map map) { + final StringBuilder sb = new StringBuilder(); + sb.append('{'); + boolean first = true; + for (final Map.Entry entry : map.entrySet()) { + if (!first) { + sb.append(','); + } + first = false; + appendJsonString(sb, entry.getKey()); + sb.append(':'); + appendJsonString(sb, entry.getValue()); + } + sb.append('}'); + return sb.toString(); + } + + private static String toJsonValue(final Object value) { + final StringBuilder sb = new StringBuilder(); + appendJsonValue(sb, value); + return sb.toString(); + } + + @SuppressWarnings("unchecked") + private static void appendJsonValue(final StringBuilder sb, final Object value) { + if (value == null) { + sb.append("null"); + } else if (value instanceof Map) { + sb.append('{'); + boolean first = true; + for (final Map.Entry entry : ((Map) value).entrySet()) { + if (!first) { + sb.append(','); + } + first = false; + appendJsonString(sb, String.valueOf(entry.getKey())); + sb.append(':'); + appendJsonValue(sb, entry.getValue()); + } + sb.append('}'); + } else if (value instanceof Iterable) { + sb.append('['); + boolean first = true; + for (final Object element : (Iterable) value) { + if (!first) { + sb.append(','); + } + first = false; + appendJsonValue(sb, element); + } + sb.append(']'); + } else if (value instanceof CharSequence || value instanceof Character) { + appendJsonString(sb, value.toString()); + } else if (value instanceof Number || value instanceof Boolean) { + sb.append(String.valueOf(value)); + } else { + appendJsonString(sb, String.valueOf(value)); + } + } + + private static void appendJsonString(final StringBuilder sb, final String value) { + sb.append('"'); + for (int i = 0; i < value.length(); i++) { + final char c = value.charAt(i); + switch (c) { + case '"': + sb.append("\\\""); + break; + case '\\': + sb.append("\\\\"); + break; + case '\n': + sb.append("\\n"); + break; + case '\r': + sb.append("\\r"); + break; + case '\t': + sb.append("\\t"); + break; + case '\b': + sb.append("\\b"); + break; + case '\f': + sb.append("\\f"); + break; + default: + if (c < 0x20) { + sb.append(String.format("\\u%04x", (int) c)); + } else { + sb.append(c); + } + } + } + sb.append('"'); + } + + // ---- test-only accessors ---- + + synchronized Set serialIdsView() { + return new TreeSet<>(serialIds); + } + + synchronized int subjectCount() { + return subjects.size(); + } + + synchronized int defaultCount() { + return defaults.size(); + } +} diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java new file mode 100644 index 00000000000..78943ed6cb5 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java @@ -0,0 +1,134 @@ +package datadog.trace.api.openfeature; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import dev.openfeature.sdk.EvaluationContext; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.Hook; +import dev.openfeature.sdk.HookContext; +import dev.openfeature.sdk.ImmutableMetadata; +import java.util.Map; + +/** + * OpenFeature {@code finally} hook that captures feature-flag evaluation metadata into + * per-local-root span state for APM span enrichment (JAVA-01). This is the CAPTURE half of the + * capture-vs-write split — the WRITE half is {@link SpanEnrichmentInterceptor}, which flushes the + * accumulated tags onto the local root span when the trace completes. + * + *

Mirrors {@link FlagEvalHook}: registered via {@link Provider#getProviderHooks()} (only when + * the span-enrichment gate is on) and reading {@code details.getFlagMetadata()}. It resolves the + * active local-root span via {@link AgentTracer#activeSpan()} and keys the shared {@link + * SpanEnrichmentAccumulator#STATES} map by that root's trace id, so an interceptor running later on + * the write thread can recover the same state from the completed span collection. + * + *

Capture branch (frozen Node reference): + * + *

    + *
  • serial id present → addSerialId, plus addSubject when {@code __dd_do_log} AND a targeting + * key + *
  • else variant missing (runtime default) → addDefault(flagKey, value) + *
+ * + *

All work is wrapped in try/catch — enrichment must NEVER break flag evaluation (Pattern D). + */ +class SpanEnrichmentHook implements Hook { + + static final String METADATA_SERIAL_ID = "__dd_split_serial_id"; + static final String METADATA_DO_LOG = "__dd_do_log"; + + /** + * Resolves the local-root span for the active trace. Injectable so tests need no static mocks. + */ + interface RootSpanResolver { + AgentSpan activeLocalRoot(); + } + + private static final RootSpanResolver DEFAULT_RESOLVER = + () -> { + final AgentSpan active = AgentTracer.activeSpan(); + if (active == null) { + return null; + } + final AgentSpan localRoot = active.getLocalRootSpan(); + return localRoot != null ? localRoot : active; + }; + + private final RootSpanResolver rootSpanResolver; + + SpanEnrichmentHook() { + this(DEFAULT_RESOLVER); + } + + SpanEnrichmentHook(final RootSpanResolver rootSpanResolver) { + this.rootSpanResolver = rootSpanResolver; + } + + @Override + public void finallyAfter( + final HookContext ctx, + final FlagEvaluationDetails details, + final Map hints) { + if (details == null) { + return; + } + try { + final AgentSpan root = rootSpanResolver.activeLocalRoot(); + if (root == null || root.getTraceId() == null) { + return; // no active span → nothing to enrich + } + final long traceKey = root.getTraceId().toLong(); + capture(traceKey, ctx, details); + } catch (final Throwable t) { + // Never let span enrichment break flag evaluation. + } + } + + /** + * Applies the frozen Node capture branch against the state keyed by {@code traceKey}. Package + * private so it can be driven deterministically in tests without stubbing the static tracer. + */ + void capture( + final long traceKey, + final HookContext ctx, + final FlagEvaluationDetails details) { + final ImmutableMetadata metadata = details.getFlagMetadata(); + final String serialIdStr = metadata != null ? metadata.getString(METADATA_SERIAL_ID) : null; + final String doLogStr = metadata != null ? metadata.getString(METADATA_DO_LOG) : null; + final boolean doLog = "true".equalsIgnoreCase(doLogStr); + final String targetingKey = targetingKey(ctx); + + if (serialIdStr != null) { + final int serialId; + try { + serialId = Integer.parseInt(serialIdStr); + } catch (final NumberFormatException e) { + return; // malformed serial id — drop, never break eval + } + final SpanEnrichmentAccumulator state = getOrCreateState(traceKey); + state.addSerialId(serialId); + if (doLog && targetingKey != null) { + state.addSubject(targetingKey, serialId); + } + } else if (details.getVariant() == null) { + // Runtime-default detection = MISSING VARIANT (never a reason enum). + getOrCreateState(traceKey).addDefault(details.getFlagKey(), details.getValue()); + } + } + + private static SpanEnrichmentAccumulator getOrCreateState(final long traceKey) { + return SpanEnrichmentAccumulator.STATES.computeIfAbsent( + traceKey, k -> new SpanEnrichmentAccumulator()); + } + + private static String targetingKey(final HookContext ctx) { + if (ctx == null) { + return null; + } + final EvaluationContext evaluationContext = ctx.getCtx(); + if (evaluationContext == null) { + return null; + } + final String key = evaluationContext.getTargetingKey(); + return (key == null || key.isEmpty()) ? null : key; + } +} diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java new file mode 100644 index 00000000000..1aef15e6e6f --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java @@ -0,0 +1,96 @@ +package datadog.trace.api.openfeature; + +import datadog.trace.api.interceptor.MutableSpan; +import datadog.trace.api.interceptor.TraceInterceptor; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import java.util.Collection; +import java.util.Map; + +/** + * {@link TraceInterceptor} that writes the accumulated {@code ffe_*} tags onto the local root span + * when a trace completes (JAVA-01). This is the WRITE half of the capture-vs-write split — the + * CAPTURE half is {@link SpanEnrichmentHook}, which fills {@link SpanEnrichmentAccumulator#STATES} + * during flag evaluation. + * + *

{@code onTraceComplete} finds the local root span in the collection (single pass — the root is + * reachable via {@link MutableSpan#getLocalRootSpan()}), reads the accumulator keyed by that root's + * trace id, writes the {@code ffe_*} tags, and removes the state. Span order is not guaranteed, so + * the root is resolved through {@code getLocalRootSpan()} rather than by position. + * + *

The state is cleared on flush regardless of whether it had data, so a trace that captured + * nothing (or whose accumulator was never created) leaves no residue (DG-005). All work is wrapped + * in try/catch — enrichment must NEVER break trace finish (Pattern D). + */ +final class SpanEnrichmentInterceptor implements TraceInterceptor { + + /** + * Unique priority in the "trace data enrichment" band, after {@code GIT_METADATA} (3) and before + * the custom-sampling band ({@code Integer.MAX_VALUE - 2}). Distinct from every value in {@code + * AbstractTraceInterceptor.Priority} and from the CI Visibility interceptors. + */ + static final int PRIORITY = 4; + + // The tracer holds interceptors in an add-only sorted set keyed by priority with no public + // removal API. On provider shutdown we cannot un-register, so we disable the singleton instead: + // a disabled interceptor no-ops and drains any residual state (Pitfall 3 — provider-close + // cleanup; idempotent because re-registration with the same priority is rejected). + private volatile boolean enabled = true; + + /** Disables the interceptor and clears all accumulated state (provider-close cleanup). */ + void disable() { + enabled = false; + SpanEnrichmentAccumulator.STATES.clear(); + } + + boolean isEnabled() { + return enabled; + } + + @Override + public Collection onTraceComplete( + final Collection trace) { + try { + if (!enabled || trace == null || trace.isEmpty()) { + return trace; + } + final MutableSpan localRoot = findLocalRoot(trace); + if (!(localRoot instanceof AgentSpan)) { + return trace; // cannot resolve trace id without AgentSpan; nothing to flush + } + final long traceKey = ((AgentSpan) localRoot).getTraceId().toLong(); + final SpanEnrichmentAccumulator state = SpanEnrichmentAccumulator.STATES.remove(traceKey); + if (state == null || !state.hasData()) { + return trace; + } + for (final Map.Entry tag : state.toSpanTags().entrySet()) { + final String value = tag.getValue(); + if (value != null && !value.isEmpty()) { + localRoot.setTag(tag.getKey(), value); + } + } + } catch (final Throwable t) { + // Never let span enrichment break trace finish. + } + return trace; + } + + private static MutableSpan findLocalRoot(final Collection trace) { + final MutableSpan first = trace.iterator().next(); + final MutableSpan localRoot = first.getLocalRootSpan(); + if (localRoot != null) { + return localRoot; + } + // Fallback: scan for a span that is its own local root (no parent in this fragment). + for (final MutableSpan span : trace) { + if (span.getLocalRootSpan() == span || span.getLocalRootSpan() == null) { + return span; + } + } + return first; + } + + @Override + public int priority() { + return PRIORITY; + } +} diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java new file mode 100644 index 00000000000..5e70c29e36d --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java @@ -0,0 +1,119 @@ +package datadog.trace.api.openfeature; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +/** + * ULEB128 delta-varint + base64 codec for APM feature-flag span enrichment (JAVA-01). + * + *

Ported VERBATIM from the frozen Node reference ({@code dd-trace-js#8343}). The tag names, + * encoding, and golden vectors are FROZEN against that contract — backend/Trino decode and the + * parametric system-tests assertions depend on exact parity, so this MUST NOT be re-derived. + * + *

Algorithm: dedupe (via {@link Set}) → sort ascending → emit the delta from the previous id as + * an unsigned LEB128 varint (7 bits/byte, MSB = continuation) → base64-encode the byte buffer. The + * empty set encodes to the empty string (the caller then omits the tag). + * + *

Golden vector: {@code {100, 108, 128, 130}} → deltas {@code [100, 8, 20, 2]} → bytes {@code + * [0x64, 0x08, 0x14, 0x02]} → base64 {@code "ZAgUAg=="}. + */ +final class ULeb128Encoder { + + private ULeb128Encoder() {} + + /** + * ULEB128 delta-varint encodes the given serial ids into a bare base64 string. + * + * @param serialIds the serial ids to encode (deduped + sorted ascending internally) + * @return the base64-encoded delta-varint bytes, or the empty string when {@code serialIds} is + * empty or null + */ + static String encodeDeltaVarint(final Set serialIds) { + if (serialIds == null || serialIds.isEmpty()) { + // Empty set encodes to the empty string; the caller omits the tag. + return ""; + } + final SortedSet sorted = + serialIds instanceof SortedSet ? (SortedSet) serialIds : new TreeSet<>(serialIds); + // Worst case: 5 bytes per 32-bit varint. + final byte[] buffer = new byte[sorted.size() * 5]; + int length = 0; + int previous = 0; + for (final Integer id : sorted) { + long delta = + ((long) id) - previous; // long to stay safe; deltas are non-negative (sorted asc) + previous = id; + while (delta > 0x7FL) { + buffer[length++] = (byte) ((delta & 0x7FL) | 0x80L); + delta >>>= 7; + } + buffer[length++] = (byte) (delta & 0x7FL); + } + final byte[] out = new byte[length]; + System.arraycopy(buffer, 0, out, 0, length); + return Base64.getEncoder().encodeToString(out); + } + + /** + * Decodes a delta-varint base64 string back into the original ascending serial ids. Used by the + * codec round-trip oracle (the decode side mirrors the L2 system-tests decoder). + * + * @param encoded a base64 string produced by {@link #encodeDeltaVarint(Set)} + * @return the decoded serial ids in ascending order (empty for the empty string) + */ + static SortedSet decodeDeltaVarint(final String encoded) { + final SortedSet result = new TreeSet<>(); + if (encoded == null || encoded.isEmpty()) { + return result; + } + final byte[] bytes = Base64.getDecoder().decode(encoded); + int previous = 0; + int index = 0; + while (index < bytes.length) { + long value = 0; + int shift = 0; + while (true) { + final byte b = bytes[index++]; + value |= ((long) (b & 0x7F)) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + previous += (int) value; // delta from previous + result.add(previous); + } + return result; + } + + /** + * Lower-case hex SHA-256 of the given string. Used to hash subject targeting keys before they are + * emitted (privacy: subject keys are never emitted in clear text). + * + * @param value the value to hash + * @return the lower-case hex SHA-256 digest + */ + static String hashTargetingKey(final String value) { + try { + final MessageDigest digest = MessageDigest.getInstance("SHA-256"); + final byte[] hash = digest.digest(value.getBytes(StandardCharsets.UTF_8)); + final StringBuilder hex = new StringBuilder(hash.length * 2); + for (final byte b : hash) { + final int v = b & 0xFF; + if (v < 0x10) { + hex.append('0'); + } + hex.append(Integer.toHexString(v)); + } + return hex.toString(); + } catch (final NoSuchAlgorithmException e) { + // SHA-256 is mandated by the JLS to be present on every JVM; this is unreachable in practice. + throw new IllegalStateException("SHA-256 algorithm not available", e); + } + } +} From 44f51c5b007b72489c5b8c7ffb71ce9aa120f72a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 15 Jun 2026 23:24:28 -0400 Subject: [PATCH 03/18] test(02-02): JUnit 5 L0 suite for span enrichment (7 required cases + golden round-trip) - Codec golden-vector round-trip {100,108,128,130} -> ZAgUAg== (encode + decode + empty + structural dedupe) - no-span, finished-root (accumulate+dedupe+flush), runtime-default missing-variant (ffe_runtime_defaults JSON object), per-subject cap (10 subj / 20 exp/subj) + doLog gating, max-200 serial ids, object-default JSON + 64-char truncation - gate-off negative control: no hook/interceptor constructed, not in getProviderHooks, no accumulator state (DG-005) - gate-on construction + provider-close cleanup (shutdown disables interceptor + drains state); error isolation; interceptor priority uniqueness - JUnit 5 only (no Groovy); injectable RootSpanResolver + Provider gate override avoid static mocking (project mockito uses the subclass mock maker) --- .../openfeature/SpanEnrichmentHookTest.java | 369 ++++++++++++++++++ 1 file changed, 369 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java new file mode 100644 index 00000000000..50938648124 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java @@ -0,0 +1,369 @@ +package datadog.trace.api.openfeature; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.trace.api.DDTraceId; +import datadog.trace.api.interceptor.MutableSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.FlagValueType; +import dev.openfeature.sdk.Hook; +import dev.openfeature.sdk.HookContext; +import dev.openfeature.sdk.ImmutableContext; +import dev.openfeature.sdk.ImmutableMetadata; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * L0 unit suite for APM feature-flag span enrichment (JAVA-01). + * + *

Covers the seven required VALIDATION.md cases plus the explicit max-200 case and the codec + * golden-vector round-trip. The contract (encoding, limits, tag shapes) is FROZEN against the Node + * reference ({@code dd-trace-js#8343}). + */ +class SpanEnrichmentHookTest { + + @BeforeEach + @AfterEach + void clearState() { + SpanEnrichmentAccumulator.STATES.clear(); + } + + // ---- helpers ---- + + private static FlagEvaluationDetails details( + final String flagKey, + final String variant, + final Object value, + final ImmutableMetadata metadata) { + return FlagEvaluationDetails.builder() + .flagKey(flagKey) + .variant(variant) + .value(value) + .flagMetadata(metadata) + .build(); + } + + private static ImmutableMetadata metadata(final Integer serialId, final boolean doLog) { + final ImmutableMetadata.ImmutableMetadataBuilder builder = ImmutableMetadata.builder(); + if (serialId != null) { + builder.addString(SpanEnrichmentHook.METADATA_SERIAL_ID, serialId.toString()); + } + builder.addString(SpanEnrichmentHook.METADATA_DO_LOG, String.valueOf(doLog)); + return builder.build(); + } + + private static HookContext ctx(final String flagKey, final String targetingKey) { + return HookContext.from( + flagKey, + FlagValueType.STRING, + null, + null, + targetingKey == null ? new ImmutableContext() : new ImmutableContext(targetingKey), + "default"); + } + + /** Drives the capture branch directly (no static tracer) for a fixed trace key. */ + private static void capture( + final SpanEnrichmentHook hook, + final long traceKey, + final String flagKey, + final String targetingKey, + final String variant, + final Object value, + final Integer serialId, + final boolean doLog) { + hook.capture( + traceKey, + ctx(flagKey, targetingKey), + details(flagKey, variant, value, metadata(serialId, doLog))); + } + + private static MutableSpan rootSpanCollection(final long traceId, final MutableSpan rootSpan) { + when(rootSpan.getLocalRootSpan()).thenReturn(rootSpan); + when(((AgentSpan) rootSpan).getTraceId()).thenReturn(DDTraceId.from(traceId)); + return rootSpan; + } + + // ---- 1. codec golden-vector round-trip ---- + + @Test + void codecGoldenVectorAndRoundTrip() { + final SortedSet ids = new TreeSet<>(); + ids.add(100); + ids.add(108); + ids.add(128); + ids.add(130); + final String encoded = ULeb128Encoder.encodeDeltaVarint(ids); + assertEquals("ZAgUAg==", encoded, "golden vector must match the frozen Node contract"); + // round-trip: decode back to the same ascending ids + assertEquals(ids, ULeb128Encoder.decodeDeltaVarint(encoded)); + // empty set -> empty string (tag omitted) + assertEquals("", ULeb128Encoder.encodeDeltaVarint(Collections.emptySet())); + // dedupe is structural: a duplicate id does not change the encoding + final SortedSet withDup = new TreeSet<>(ids); + withDup.add(100); + assertEquals(encoded, ULeb128Encoder.encodeDeltaVarint(withDup)); + } + + // ---- 2. no-span (no active root) ---- + + @Test + void noActiveSpanDoesNotCrashOrAccumulate() { + // Injected resolver returns null => no active local root (the no-span case). + final SpanEnrichmentHook hook = new SpanEnrichmentHook(() -> null); + // Should be a no-op, never throw. + hook.finallyAfter( + ctx("flag", "user-1"), + details("flag", "on", "v", metadata(100, true)), + Collections.emptyMap()); + assertTrue( + SpanEnrichmentAccumulator.STATES.isEmpty(), "no active span => no accumulator state"); + } + + @Test + void finallyAfterResolvesRootViaResolverAndAccumulates() { + // Drives finallyAfter end-to-end with an injected root resolver (no static mocks). + final AgentSpan root = mock(AgentSpan.class); + when(root.getTraceId()).thenReturn(DDTraceId.from(0x77L)); + final SpanEnrichmentHook hook = new SpanEnrichmentHook(() -> root); + hook.finallyAfter( + ctx("flag", "user-1"), + details("flag", "on", "v", metadata(42, true)), + Collections.emptyMap()); + final SpanEnrichmentAccumulator state = SpanEnrichmentAccumulator.STATES.get(0x77L); + assertTrue(state != null && state.serialIdsView().contains(42)); + assertEquals(1, state.subjectCount(), "doLog=true + targeting key => subject recorded"); + } + + // ---- 3. finished-root (accumulate + dedupe, then flush via interceptor) ---- + + @Test + void finishedRootFlushesFlagsEncTag() { + final SpanEnrichmentHook hook = new SpanEnrichmentHook(); + final long traceId = 0xABCDL; + // accumulate {100,108,128,130} with a duplicate 100 to prove dedupe + capture(hook, traceId, "f1", "user-1", "on", "v", 100, false); + capture(hook, traceId, "f2", "user-1", "on", "v", 108, false); + capture(hook, traceId, "f3", "user-1", "on", "v", 128, false); + capture(hook, traceId, "f4", "user-1", "on", "v", 130, false); + capture(hook, traceId, "f1", "user-1", "on", "v", 100, false); // dup + + final AgentSpan root = mock(AgentSpan.class); + rootSpanCollection(traceId, root); + final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(); + + interceptor.onTraceComplete(Collections.singletonList(root)); + + verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZAgUAg=="); + // state cleared after flush + assertTrue(SpanEnrichmentAccumulator.STATES.isEmpty(), "state must be cleared on flush"); + } + + // ---- 4. error/default variant (missing variant -> ffe_runtime_defaults JSON object) ---- + + @Test + void runtimeDefaultMissingVariantWritesJsonObject() { + final SpanEnrichmentHook hook = new SpanEnrichmentHook(); + final long traceId = 0x1234L; + // no serial id + null variant => runtime default; object value must be JSON-stringified + final Map objectValue = Collections.singletonMap("k", "val"); + hook.capture( + traceId, + ctx("obj-flag", "user-1"), + FlagEvaluationDetails.builder() + .flagKey("obj-flag") + .variant(null) + .value(objectValue) + .flagMetadata(ImmutableMetadata.builder().build()) + .build()); + + final AgentSpan root = mock(AgentSpan.class); + rootSpanCollection(traceId, root); + new SpanEnrichmentInterceptor().onTraceComplete(Collections.singletonList(root)); + + // ffe_runtime_defaults is a JSON object string, NOT [object Object]/toString. + verify(root) + .setTag( + SpanEnrichmentAccumulator.TAG_RUNTIME_DEFAULTS, + "{\"obj-flag\":\"{\\\"k\\\":\\\"val\\\"}\"}"); + // no flags tag when there are no serial ids + verify(root, never()).setTag(eq(SpanEnrichmentAccumulator.TAG_FLAGS_ENC), anyString()); + } + + // ---- 5. per-subject cap (10 subjects / 20 experiments / doLog gating) ---- + + @Test + void subjectCapsAndDoLogGating() { + final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); + + // doLog gating: addSubject only happens when doLog true (driven via the hook branch below). + final SpanEnrichmentHook hook = new SpanEnrichmentHook(); + final long traceId = 0x5L; + capture(hook, traceId, "f", "user-A", "on", "v", 1, false); // doLog=false => no subject + assertEquals( + 0, + SpanEnrichmentAccumulator.STATES.get(traceId).subjectCount(), + "doLog=false must not record a subject"); + capture(hook, traceId, "f", "user-A", "on", "v", 2, true); // doLog=true => subject recorded + assertEquals(1, SpanEnrichmentAccumulator.STATES.get(traceId).subjectCount()); + + // per-subject experiment cap: 20 max + for (int i = 0; i < 25; i++) { + acc.addSubject("subjectX", i); + } + final Map tags = acc.toSpanTags(); + final SortedSet decoded = + ULeb128Encoder.decodeDeltaVarint( + // ffe_subjects_enc is {"":""}; extract the single base64 value + tags.get(SpanEnrichmentAccumulator.TAG_SUBJECTS_ENC) + .replaceAll("^\\{\"[a-f0-9]+\":\"", "") + .replaceAll("\"\\}$", "")); + assertEquals(SpanEnrichmentAccumulator.MAX_EXPERIMENTS_PER_SUBJECT, decoded.size()); + + // subject cap: 10 max distinct subjects + final SpanEnrichmentAccumulator acc2 = new SpanEnrichmentAccumulator(); + for (int i = 0; i < 15; i++) { + acc2.addSubject("subject-" + i, i); + } + assertEquals(SpanEnrichmentAccumulator.MAX_SUBJECTS, acc2.subjectCount()); + } + + // ---- 6. max-200 serial ids ---- + + @Test + void max200SerialIdsEnforced() { + final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); + for (int i = 0; i < 300; i++) { + acc.addSerialId(i); + } + assertEquals( + SpanEnrichmentAccumulator.MAX_SERIAL_IDS, + acc.serialIdsView().size(), + "serial ids must be capped at 200"); + } + + // ---- 7. JSON/object-default (json not toString + 64-char truncation) ---- + + @Test + void objectDefaultJsonAndTruncation() { + // object -> JSON, not toString + assertEquals( + "{\"a\":\"b\"}", + SpanEnrichmentAccumulator.stringifyDefault(Collections.singletonMap("a", "b"))); + // scalar string -> as-is + assertEquals("hello", SpanEnrichmentAccumulator.stringifyDefault("hello")); + // null -> "null" + assertEquals("null", SpanEnrichmentAccumulator.stringifyDefault(null)); + + // 64-char truncation (first-wins handled separately) + final StringBuilder longValue = new StringBuilder(); + for (int i = 0; i < 100; i++) { + longValue.append('x'); + } + final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); + acc.addDefault("flag", longValue.toString()); + final String tag = acc.toSpanTags().get(SpanEnrichmentAccumulator.TAG_RUNTIME_DEFAULTS); + // {"flag":"<64 x's>"} + final String expectedValue = + longValue.substring(0, SpanEnrichmentAccumulator.MAX_DEFAULT_VALUE_LENGTH); + assertEquals("{\"flag\":\"" + expectedValue + "\"}", tag); + + // first-wins: a second addDefault for the same flag is ignored + acc.addDefault("flag", "second"); + assertEquals(1, acc.defaultCount()); + } + + // ---- gate-off negative control (no ffe_*, no hook/interceptor, no state) (DG-005) ---- + + @Test + void gateOffConstructsNothingAndAccumulatesNoState() { + // Gate OFF via the injectable override (no static config mocking). + final Provider provider = new Provider(new Provider.Options(), null, Boolean.FALSE); + + // No hook, no interceptor constructed (DG-005 zero-idle-overhead). + assertNull(provider.spanEnrichmentHook(), "gate off => no span-enrichment hook"); + assertNull(provider.spanEnrichmentInterceptor(), "gate off => no span-enrichment interceptor"); + // getProviderHooks must not contain a SpanEnrichmentHook. + final List hooks = provider.getProviderHooks(); + for (final Hook hook : hooks) { + assertFalse( + hook instanceof SpanEnrichmentHook, "gate off => SpanEnrichmentHook never registered"); + } + // No accumulator state created. + assertTrue(SpanEnrichmentAccumulator.STATES.isEmpty(), "gate off => no accumulator state"); + } + + // ---- gate-on construction + provider-close cleanup ---- + + @Test + void gateOnConstructsHookAndInterceptorThenShutdownDisables() { + // Gate ON via the injectable override. + final Provider provider = new Provider(new Provider.Options(), null, Boolean.TRUE); + assertTrue(provider.spanEnrichmentHook() != null, "gate on => hook constructed"); + assertTrue(provider.spanEnrichmentInterceptor() != null, "gate on => interceptor constructed"); + assertTrue(provider.spanEnrichmentInterceptor().isEnabled()); + // hook registered in provider hooks + boolean registered = false; + for (final Hook hook : provider.getProviderHooks()) { + if (hook instanceof SpanEnrichmentHook) { + registered = true; + } + } + assertTrue(registered, "gate on => SpanEnrichmentHook registered in getProviderHooks"); + + // provider close disables the interceptor (provider-close cleanup) and drains state + SpanEnrichmentAccumulator.STATES.put(1L, new SpanEnrichmentAccumulator()); + provider.shutdown(); + assertFalse(provider.spanEnrichmentInterceptor().isEnabled(), "shutdown disables interceptor"); + assertTrue(SpanEnrichmentAccumulator.STATES.isEmpty(), "shutdown drains residual state"); + } + + // ---- error isolation: enrichment never throws ---- + + @Test + void captureNeverThrowsOnNullInputs() { + final SpanEnrichmentHook hook = new SpanEnrichmentHook(); + // null details handled in finallyAfter; capture with empty metadata + null variant + hook.finallyAfter(null, null, null); + assertTrue(SpanEnrichmentAccumulator.STATES.isEmpty()); + } + + // ---- interceptor gate-off / empty trace robustness ---- + + @Test + void interceptorNoOpsWhenDisabledOrEmpty() { + final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(); + // empty trace + assertTrue(interceptor.onTraceComplete(Collections.emptyList()).isEmpty()); + // disabled + interceptor.disable(); + final AgentSpan root = mock(AgentSpan.class); + final List trace = Collections.singletonList(root); + interceptor.onTraceComplete(trace); + verify(root, never()).setTag(anyString(), anyString()); + } + + // ---- interceptor priority uniqueness ---- + + @Test + void interceptorPriorityIsUnique() { + // Distinct from AbstractTraceInterceptor.Priority values (0,1,2,3, MAX-2, MAX-1, MAX). + assertEquals(4, new SpanEnrichmentInterceptor().priority()); + } +} From 545f72b2e869cb8766db341bdb69f3c8f3446c33 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 00:37:04 -0400 Subject: [PATCH 04/18] fix(02-02): correct span-enrichment lifecycle (partial-flush, leak, reconfig) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gap-closure for the three BLOCKER lifecycle defects in 02-REVIEW-java.md. The frozen contract (codec/limits/gate/tag shapes, golden vector ZAgUAg==) is unchanged — this fixes only how per-trace state is scoped, bounded, and flushed. CR-01 (partial-flush data loss + tag misattribution): CoreTracer.write -> interceptCompleteTrace runs onTraceComplete on EVERY flush, and PendingTrace.write(isPartial) excludes the still-open local root from a partial flush (getRootSpan() is null until the final write). The interceptor previously resolved the root by reference (reachable even when absent from the fragment) and unconditionally removed the accumulator on the first partial flush, dropping all pre-flush flags and tagging a not-yet-finished root. SpanEnrichmentInterceptor now flushes + removes ONLY when the local root is actually present in the flushed collection (the final write); a partial flush leaves the accumulator intact. Also adds the missing getTraceId() null guard (WR-01) and never falls back to tagging a non-root span (WR-02). CR-02 (unbounded state leak): State lived in a static ConcurrentHashMap whose only removal paths were trace-complete and shutdown, so dropped / Noop / never-finishing traces leaked forever (#4844 leak class). State now lives in a per-provider SpanEnrichmentStates store, hard-bounded at MAX_TRACES with FIFO eviction of the oldest in-flight entry, so a never-completing trace cannot grow the map unboundedly. CR-03 (reconfiguration corruption): Provider ignored addTraceInterceptor's return value, so a second gate-on provider (rejected on duplicate priority 4) still wired a hook into shared static state and its shutdown() cleared the first provider's live state. Provider now honors the registration result (wires the hook/interceptor only when registration succeeds) and each provider owns its own state store, so one provider's shutdown can never clear another's. Removes the global static map (review IN-03 root cause). Adds an injectable TraceInterceptorRegistrar test seam (mirrors the existing gate-override seam). Runtime-default rendering (String() parity) and hasData() are intentionally unchanged. L0 suite migrated to the instance-owned store; module 146 tests green. --- .../trace/api/openfeature/Provider.java | 60 +++++++++- .../SpanEnrichmentAccumulator.java | 18 +-- .../api/openfeature/SpanEnrichmentHook.java | 28 +++-- .../SpanEnrichmentInterceptor.java | 108 +++++++++++++---- .../api/openfeature/SpanEnrichmentStates.java | 112 ++++++++++++++++++ .../openfeature/SpanEnrichmentHookTest.java | 78 +++++++----- 6 files changed, 318 insertions(+), 86 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index 71377c8a39b..5fe38851635 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -63,6 +63,17 @@ public Provider(final Options options) { this(options, evaluator, null); } + /** + * Registers a {@link SpanEnrichmentInterceptor} with the running tracer, returning {@code true} + * when it was added and {@code false} when the tracer rejected it (e.g. a duplicate-priority + * interceptor from an earlier provider is already registered). Injectable so tests can drive the + * success and rejected-registration (reconfiguration) paths deterministically without mutating + * the global tracer (mirrors the {@code spanEnrichmentEnabledOverride} seam). + */ + interface TraceInterceptorRegistrar { + boolean register(SpanEnrichmentInterceptor interceptor); + } + /** * @param spanEnrichmentEnabledOverride when non-null, forces the span-enrichment gate (test * seam); when null, the gate is read from {@link #SPAN_ENRICHMENT_ENABLED_ENV}. @@ -71,6 +82,22 @@ public Provider(final Options options) { final Options options, final Evaluator evaluator, final Boolean spanEnrichmentEnabledOverride) { + this( + options, + evaluator, + spanEnrichmentEnabledOverride, + interceptor -> GlobalTracer.get().addTraceInterceptor(interceptor)); + } + + /** + * @param registrar registers the span-enrichment interceptor with the tracer; injectable for + * tests (see {@link TraceInterceptorRegistrar}). + */ + Provider( + final Options options, + final Evaluator evaluator, + final Boolean spanEnrichmentEnabledOverride, + final TraceInterceptorRegistrar registrar) { this.options = options; this.evaluator = evaluator; FlagEvalMetrics metrics = null; @@ -96,9 +123,30 @@ public Provider(final Options options) { SpanEnrichmentInterceptor seInterceptor = null; if (spanEnrichmentEnabled) { try { - seHook = new SpanEnrichmentHook(); - seInterceptor = new SpanEnrichmentInterceptor(); - GlobalTracer.get().addTraceInterceptor(seInterceptor); + // Per-provider state store (NOT a global static): owned by the interceptor, shared with the + // hook, so this provider's shutdown can never clear another provider's state (CR-03), and a + // never-completing trace cannot leak unboundedly (CR-02, bounded inside the store). + final SpanEnrichmentStates seStates = new SpanEnrichmentStates(); + final SpanEnrichmentInterceptor candidate = new SpanEnrichmentInterceptor(seStates); + // HONOR the registration result (CR-03): the tracer rejects an interceptor whose priority + // is + // already taken (e.g. a SECOND gate-on provider after reconfiguration) and returns false. + // If + // we ignored it, this provider would still wire a hook into shared state yet never have its + // interceptor invoked, and its shutdown() would clear the FIRST provider's live state. So + // we + // only wire the hook+interceptor when registration actually succeeded. + if (registrar.register(candidate)) { + seInterceptor = candidate; + seHook = new SpanEnrichmentHook(seStates); + } else { + // Duplicate registration (active interceptor already owns priority 4). Leave hook + + // interceptor null so this provider neither accumulates orphan state nor clears the + // active provider's state on shutdown. Its own seStates is unreferenced and GC'd. + log.warn( + "Span enrichment interceptor already registered (duplicate provider); " + + "skipping duplicate registration to avoid corrupting active provider state"); + } } catch (LinkageError | Exception e) { // Tracer classes absent (e.g. API-only classpath): degrade to no span enrichment. log.warn("Span enrichment unavailable — tracer classes not on classpath", e); @@ -242,7 +290,11 @@ public void shutdown() { flagEvalMetrics.shutdown(); } // Provider-close cleanup for span enrichment (Pitfall 3): the tracer has no interceptor-removal - // API, so disable the interceptor (it then no-ops + drains residual state) and drop the hook. + // API, so disable the interceptor (it then no-ops + drains its OWN residual state). Because the + // state store is instance-owned, this clears only this provider's state and can never wipe an + // active second provider's in-flight state (CR-03). When this provider's registration was + // rejected as a duplicate, spanEnrichmentInterceptor is null here, so shutdown is a no-op and + // the active provider's state is untouched. if (spanEnrichmentInterceptor != null) { spanEnrichmentInterceptor.disable(); } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java index 655594c153b..df37c42bf37 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java @@ -4,7 +4,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.ConcurrentHashMap; /** * Per-local-root-span accumulator for APM feature-flag span enrichment (JAVA-01). @@ -14,10 +13,11 @@ * shapes are FROZEN against the Node reference ({@code dd-trace-js#8343}) — see {@link * ULeb128Encoder}. * - *

Instances are created lazily and stored in {@link #STATES}, keyed by the local-root span's - * trace id. The capture hook ({@link SpanEnrichmentHook}) writes; the write interceptor ({@link - * SpanEnrichmentInterceptor}) reads and clears. When the span-enrichment gate is off, nothing - * touches {@link #STATES}, so there is no idle per-span overhead (DG-005). + *

Instances are created lazily and held in a per-provider {@link SpanEnrichmentStates} store, + * keyed by the local-root span's trace id. The capture hook ({@link SpanEnrichmentHook}) writes; + * the write interceptor ({@link SpanEnrichmentInterceptor}) reads and clears. When the + * span-enrichment gate is off, no store and no accumulator are ever created, so there is no idle + * per-span overhead (DG-005). * *

Output tag shapes (Pattern F): * @@ -39,14 +39,6 @@ final class SpanEnrichmentAccumulator { static final String TAG_SUBJECTS_ENC = "ffe_subjects_enc"; static final String TAG_RUNTIME_DEFAULTS = "ffe_runtime_defaults"; - /** - * Shared per-trace state, keyed by the local-root span's trace id ({@code DDTraceId.toLong()}). - * Capture (hook) puts; write (interceptor) reads + removes. Only ever populated when the gate is - * on (DG-005). - */ - static final ConcurrentHashMap STATES = - new ConcurrentHashMap<>(); - // dedupe is structural (a Set); sorted for deterministic encoding. private final TreeSet serialIds = new TreeSet<>(); // sha256hex(targetingKey) -> serial ids. LinkedHashMap for stable iteration order. diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java index 78943ed6cb5..42d1c4eef61 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java @@ -17,9 +17,11 @@ * *

Mirrors {@link FlagEvalHook}: registered via {@link Provider#getProviderHooks()} (only when * the span-enrichment gate is on) and reading {@code details.getFlagMetadata()}. It resolves the - * active local-root span via {@link AgentTracer#activeSpan()} and keys the shared {@link - * SpanEnrichmentAccumulator#STATES} map by that root's trace id, so an interceptor running later on - * the write thread can recover the same state from the completed span collection. + * active local-root span via {@link AgentTracer#activeSpan()} and keys the per-provider {@link + * SpanEnrichmentStates} store (shared with this provider's {@link SpanEnrichmentInterceptor}) by + * that root's trace id, so the interceptor running later on the write thread can recover the same + * state from the completed span collection. The store is owned by the interceptor (not a global + * static), so a second provider can never clear this one's state (CR-03). * *

Capture branch (frozen Node reference): * @@ -54,13 +56,18 @@ interface RootSpanResolver { }; private final RootSpanResolver rootSpanResolver; + // Per-provider state store, shared with this provider's interceptor (NOT a global static). The + // hook writes here; the interceptor reads + removes. Owning the store on the interceptor is what + // makes one provider's shutdown unable to clear another's in-flight state (CR-03). + private final SpanEnrichmentStates states; - SpanEnrichmentHook() { - this(DEFAULT_RESOLVER); + SpanEnrichmentHook(final SpanEnrichmentStates states) { + this(DEFAULT_RESOLVER, states); } - SpanEnrichmentHook(final RootSpanResolver rootSpanResolver) { + SpanEnrichmentHook(final RootSpanResolver rootSpanResolver, final SpanEnrichmentStates states) { this.rootSpanResolver = rootSpanResolver; + this.states = states; } @Override @@ -104,22 +111,17 @@ void capture( } catch (final NumberFormatException e) { return; // malformed serial id — drop, never break eval } - final SpanEnrichmentAccumulator state = getOrCreateState(traceKey); + final SpanEnrichmentAccumulator state = states.getOrCreate(traceKey); state.addSerialId(serialId); if (doLog && targetingKey != null) { state.addSubject(targetingKey, serialId); } } else if (details.getVariant() == null) { // Runtime-default detection = MISSING VARIANT (never a reason enum). - getOrCreateState(traceKey).addDefault(details.getFlagKey(), details.getValue()); + states.getOrCreate(traceKey).addDefault(details.getFlagKey(), details.getValue()); } } - private static SpanEnrichmentAccumulator getOrCreateState(final long traceKey) { - return SpanEnrichmentAccumulator.STATES.computeIfAbsent( - traceKey, k -> new SpanEnrichmentAccumulator()); - } - private static String targetingKey(final HookContext ctx) { if (ctx == null) { return null; diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java index 1aef15e6e6f..24aefc14ca9 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java @@ -1,5 +1,6 @@ package datadog.trace.api.openfeature; +import datadog.trace.api.DDTraceId; import datadog.trace.api.interceptor.MutableSpan; import datadog.trace.api.interceptor.TraceInterceptor; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -8,18 +9,35 @@ /** * {@link TraceInterceptor} that writes the accumulated {@code ffe_*} tags onto the local root span - * when a trace completes (JAVA-01). This is the WRITE half of the capture-vs-write split — the - * CAPTURE half is {@link SpanEnrichmentHook}, which fills {@link SpanEnrichmentAccumulator#STATES} - * during flag evaluation. + * when a trace actually completes (JAVA-01). This is the WRITE half of the + * capture-vs-write split — the CAPTURE half is {@link SpanEnrichmentHook}, which fills this + * interceptor's per-provider {@link SpanEnrichmentStates} store during flag evaluation. * - *

{@code onTraceComplete} finds the local root span in the collection (single pass — the root is - * reachable via {@link MutableSpan#getLocalRootSpan()}), reads the accumulator keyed by that root's - * trace id, writes the {@code ffe_*} tags, and removes the state. Span order is not guaranteed, so - * the root is resolved through {@code getLocalRootSpan()} rather than by position. + *

Partial-flush correctness (CR-01)

* - *

The state is cleared on flush regardless of whether it had data, so a trace that captured - * nothing (or whose accumulator was never created) leaves no residue (DG-005). All work is wrapped - * in try/catch — enrichment must NEVER break trace finish (Pattern D). + *

{@code dd-trace-core} runs {@code onTraceComplete} on every flush, not only on final + * trace completion: {@code CoreTracer.write(SpanList)} →{@code interceptCompleteTrace(...)} fires + * for both partial flushes ({@code PendingTrace.partialFlush()} → {@code write(true)}) and the + * final write ({@code write(false)}). A partial flush deliberately excludes the + * still-open local root — {@code PendingTrace} holds the root back ({@code rootSpanWritten}) + * and {@code getRootSpan()} returns null on a partial fragment, so {@code CoreTracer.write} does + * not invoke {@code onRootSpanFinished} for it. + * + *

Therefore this interceptor flushes+removes state only when the local root span is present + * in the flushed collection (i.e. this is the final write for the trace). On a partial flush + * the root is absent, so we return early and keep the accumulator intact, preserving every + * flag evaluated before the flush boundary. Without this guard, the first partial flush would drain + * the accumulator and write tags onto a not-yet-finished root, silently dropping all pre-flush + * enrichment — exactly the long-running-trace data-loss bug. + * + *

State ownership (CR-02 / CR-03)

+ * + *

State lives in a per-interceptor (per-provider) {@link SpanEnrichmentStates} store, not a + * global static. {@link #disable()} clears only this instance's store, so one provider's shutdown + * can never wipe another's in-flight state (CR-03). The store is hard-bounded, so a trace that + * never reaches this interceptor cannot leak unboundedly (CR-02). + * + *

All work is wrapped in try/catch — enrichment must NEVER break trace finish (Pattern D). */ final class SpanEnrichmentInterceptor implements TraceInterceptor { @@ -31,15 +49,27 @@ final class SpanEnrichmentInterceptor implements TraceInterceptor { static final int PRIORITY = 4; // The tracer holds interceptors in an add-only sorted set keyed by priority with no public - // removal API. On provider shutdown we cannot un-register, so we disable the singleton instead: - // a disabled interceptor no-ops and drains any residual state (Pitfall 3 — provider-close - // cleanup; idempotent because re-registration with the same priority is rejected). + // removal API. On provider shutdown we cannot un-register, so we disable this instance instead: + // a disabled interceptor no-ops and drains its own residual state (Pitfall 3 — provider-close + // cleanup). Because state is instance-owned, disabling never touches another provider's state. private volatile boolean enabled = true; - /** Disables the interceptor and clears all accumulated state (provider-close cleanup). */ + // Per-provider state store, shared with this provider's hook. Owned here so cleanup is scoped. + private final SpanEnrichmentStates states; + + SpanEnrichmentInterceptor(final SpanEnrichmentStates states) { + this.states = states; + } + + /** The state store shared with this interceptor's capture hook. */ + SpanEnrichmentStates states() { + return states; + } + + /** Disables the interceptor and clears its OWN accumulated state (provider-close cleanup). */ void disable() { enabled = false; - SpanEnrichmentAccumulator.STATES.clear(); + states.clear(); } boolean isEnabled() { @@ -53,12 +83,19 @@ public Collection onTraceComplete( if (!enabled || trace == null || trace.isEmpty()) { return trace; } - final MutableSpan localRoot = findLocalRoot(trace); + // Resolve the local root for this fragment, then require that the root is actually PRESENT in + // this collection. A partial flush excludes the still-open root (CR-01), so its absence means + // "not the final write" — keep the accumulator and bail. + final MutableSpan localRoot = findLocalRootInFragment(trace); if (!(localRoot instanceof AgentSpan)) { - return trace; // cannot resolve trace id without AgentSpan; nothing to flush + return trace; // partial flush, or no resolvable in-fragment root: keep state untouched + } + final DDTraceId traceId = ((AgentSpan) localRoot).getTraceId(); + if (traceId == null) { + return trace; // no trace id (e.g. Noop span) — cannot key state; keep it (WR-01) } - final long traceKey = ((AgentSpan) localRoot).getTraceId().toLong(); - final SpanEnrichmentAccumulator state = SpanEnrichmentAccumulator.STATES.remove(traceKey); + final long traceKey = traceId.toLong(); + final SpanEnrichmentAccumulator state = states.remove(traceKey); if (state == null || !state.hasData()) { return trace; } @@ -74,19 +111,38 @@ public Collection onTraceComplete( return trace; } - private static MutableSpan findLocalRoot(final Collection trace) { + /** + * Resolves the local root span for this fragment and returns it ONLY if it is actually present in + * the fragment by reference identity. Returns {@code null} when the root is not in the collection + * (a partial flush excludes the still-open root — CR-01) or when no root can be safely + * identified. + * + *

We never guess: a non-root span is never returned (WR-02). If the first span reports a + * non-null local root, we accept it only after confirming that exact object is in the fragment; + * otherwise we look for a span that is provably its own local root and present. + */ + private static MutableSpan findLocalRootInFragment( + final Collection trace) { final MutableSpan first = trace.iterator().next(); - final MutableSpan localRoot = first.getLocalRootSpan(); - if (localRoot != null) { - return localRoot; + final MutableSpan candidate = first.getLocalRootSpan(); + if (candidate != null) { + // Accept the reported local root only if it is genuinely part of THIS fragment. On a partial + // flush the root is reachable by reference but NOT in the collection → reject (keep state). + for (final MutableSpan span : trace) { + if (span == candidate) { + return candidate; + } + } + return null; // root excluded from this fragment → partial flush, do not flush/remove } - // Fallback: scan for a span that is its own local root (no parent in this fragment). + // Local root unknown for the first span: only accept a span that is provably its own local root + // and present here. Never fall back to an arbitrary span (WR-02). for (final MutableSpan span : trace) { - if (span.getLocalRootSpan() == span || span.getLocalRootSpan() == null) { + if (span.getLocalRootSpan() == span) { return span; } } - return first; + return null; } @Override diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java new file mode 100644 index 00000000000..8bd81433c77 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java @@ -0,0 +1,112 @@ +package datadog.trace.api.openfeature; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Bounded, instance-owned store of per-trace {@link SpanEnrichmentAccumulator} state, keyed by the + * local-root span's trace id ({@code DDTraceId.toLong()}). + * + *

This replaces the previous global {@code static} map (review IN-03 / CR-02 / CR-03). Each + * {@link SpanEnrichmentInterceptor} owns exactly one instance and shares it with the {@link + * SpanEnrichmentHook} for the same provider, so: + * + *

    + *
  • CR-02 (unbounded leak): the store is hard-capped at {@link #MAX_TRACES}. A trace + * that never reaches the interceptor (dropped, Noop tracer, never-finishing root) can no + * longer leak unboundedly — once the cap is reached, the oldest in-flight entry is + * evicted (FIFO by insertion order) so the map size is strictly bounded regardless of trace + * completion. This is the exact #4844 leak class. + *
  • CR-03 (reconfiguration corruption): because the store is per-interceptor rather than + * a shared static, one provider's {@code shutdown()} clears only its own state and can never + * wipe another (still-active) provider's in-flight entries. + *
+ * + *

Bounded eviction is intentionally lossy under pathological pressure: dropping the oldest + * accumulator degrades enrichment for that one (likely already-abandoned) trace rather than + * exhausting the heap. Enrichment correctness is best-effort by contract; heap safety is not. + * + *

Thread-safety: all access is guarded by the intrinsic lock on this instance. The hook writes + * (eval thread) and the interceptor reads+removes (trace-write thread) concurrently, so every + * mutator and accessor synchronizes. Contention is low — operations are O(1) map touches. + */ +final class SpanEnrichmentStates { + + private static final Logger log = LoggerFactory.getLogger(SpanEnrichmentStates.class); + + /** + * Hard cap on the number of concurrently-tracked traces. Sized well above any realistic count of + * simultaneously in-flight traces with active flag evaluations on a single JVM, so the live path + * never evicts; the cap exists purely to bound a leak of never-completing traces (CR-02). + */ + static final int MAX_TRACES = 4096; + + // Insertion-ordered so the eldest entry is the natural eviction victim (FIFO). accessOrder=false + // (the default) — we evict by age of creation, not by recency of use, so a long-running but + // never-completing trace cannot pin the map by being repeatedly touched. + private final LinkedHashMap states = new LinkedHashMap<>(); + + // One-shot guard so a sustained leak logs once at WARN rather than on every eviction. + private boolean evictionWarned = false; + + /** + * Returns the accumulator for {@code traceKey}, creating (and inserting) it if absent. When + * insertion would exceed {@link #MAX_TRACES}, the eldest entry is evicted first so the store + * stays bounded (CR-02). + */ + synchronized SpanEnrichmentAccumulator getOrCreate(final long traceKey) { + SpanEnrichmentAccumulator existing = states.get(traceKey); + if (existing != null) { + return existing; + } + if (states.size() >= MAX_TRACES) { + evictEldest(); + } + final SpanEnrichmentAccumulator created = new SpanEnrichmentAccumulator(); + states.put(traceKey, created); + return created; + } + + /** Removes and returns the accumulator for {@code traceKey}, or {@code null} if absent. */ + synchronized SpanEnrichmentAccumulator remove(final long traceKey) { + return states.remove(traceKey); + } + + /** Clears all tracked state (provider-close cleanup). Scoped to this instance only (CR-03). */ + synchronized void clear() { + states.clear(); + } + + synchronized int size() { + return states.size(); + } + + synchronized boolean isEmpty() { + return states.isEmpty(); + } + + // ---- test-only accessor ---- + + synchronized SpanEnrichmentAccumulator peek(final long traceKey) { + return states.get(traceKey); + } + + private void evictEldest() { + final Iterator> it = states.entrySet().iterator(); + if (it.hasNext()) { + it.next(); + it.remove(); + } + if (!evictionWarned) { + evictionWarned = true; + log.warn( + "Span-enrichment state cap ({}) reached; evicting oldest in-flight trace state. " + + "This indicates traces with flag evaluations that never complete; enrichment for " + + "evicted traces is dropped to bound memory.", + MAX_TRACES); + } + } +} diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java index 50938648124..4b166fe2abf 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java @@ -38,10 +38,18 @@ */ class SpanEnrichmentHookTest { + // Per-test instance-owned state store (replaces the former global static). The hook and the + // interceptor under test share this one, mirroring how a single Provider wires them (CR-03). + private SpanEnrichmentStates states; + @BeforeEach + void freshState() { + states = new SpanEnrichmentStates(); + } + @AfterEach void clearState() { - SpanEnrichmentAccumulator.STATES.clear(); + states.clear(); } // ---- helpers ---- @@ -94,9 +102,14 @@ private static void capture( details(flagKey, variant, value, metadata(serialId, doLog))); } - private static MutableSpan rootSpanCollection(final long traceId, final MutableSpan rootSpan) { + /** + * Configures a mock root span to report itself as its own local root with the given trace id. The + * returned span, passed in a singleton collection, models a FINAL flush (the root is present in + * the fragment), so the interceptor flushes + removes state (CR-01 final-write path). + */ + private static AgentSpan rootSpanCollection(final long traceId, final AgentSpan rootSpan) { when(rootSpan.getLocalRootSpan()).thenReturn(rootSpan); - when(((AgentSpan) rootSpan).getTraceId()).thenReturn(DDTraceId.from(traceId)); + when(rootSpan.getTraceId()).thenReturn(DDTraceId.from(traceId)); return rootSpan; } @@ -126,14 +139,13 @@ void codecGoldenVectorAndRoundTrip() { @Test void noActiveSpanDoesNotCrashOrAccumulate() { // Injected resolver returns null => no active local root (the no-span case). - final SpanEnrichmentHook hook = new SpanEnrichmentHook(() -> null); + final SpanEnrichmentHook hook = new SpanEnrichmentHook(() -> null, states); // Should be a no-op, never throw. hook.finallyAfter( ctx("flag", "user-1"), details("flag", "on", "v", metadata(100, true)), Collections.emptyMap()); - assertTrue( - SpanEnrichmentAccumulator.STATES.isEmpty(), "no active span => no accumulator state"); + assertTrue(states.isEmpty(), "no active span => no accumulator state"); } @Test @@ -141,12 +153,12 @@ void finallyAfterResolvesRootViaResolverAndAccumulates() { // Drives finallyAfter end-to-end with an injected root resolver (no static mocks). final AgentSpan root = mock(AgentSpan.class); when(root.getTraceId()).thenReturn(DDTraceId.from(0x77L)); - final SpanEnrichmentHook hook = new SpanEnrichmentHook(() -> root); + final SpanEnrichmentHook hook = new SpanEnrichmentHook(() -> root, states); hook.finallyAfter( ctx("flag", "user-1"), details("flag", "on", "v", metadata(42, true)), Collections.emptyMap()); - final SpanEnrichmentAccumulator state = SpanEnrichmentAccumulator.STATES.get(0x77L); + final SpanEnrichmentAccumulator state = states.peek(0x77L); assertTrue(state != null && state.serialIdsView().contains(42)); assertEquals(1, state.subjectCount(), "doLog=true + targeting key => subject recorded"); } @@ -155,7 +167,7 @@ void finallyAfterResolvesRootViaResolverAndAccumulates() { @Test void finishedRootFlushesFlagsEncTag() { - final SpanEnrichmentHook hook = new SpanEnrichmentHook(); + final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); final long traceId = 0xABCDL; // accumulate {100,108,128,130} with a duplicate 100 to prove dedupe capture(hook, traceId, "f1", "user-1", "on", "v", 100, false); @@ -166,20 +178,20 @@ void finishedRootFlushesFlagsEncTag() { final AgentSpan root = mock(AgentSpan.class); rootSpanCollection(traceId, root); - final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(); + final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); interceptor.onTraceComplete(Collections.singletonList(root)); verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZAgUAg=="); // state cleared after flush - assertTrue(SpanEnrichmentAccumulator.STATES.isEmpty(), "state must be cleared on flush"); + assertTrue(states.isEmpty(), "state must be cleared on flush"); } // ---- 4. error/default variant (missing variant -> ffe_runtime_defaults JSON object) ---- @Test void runtimeDefaultMissingVariantWritesJsonObject() { - final SpanEnrichmentHook hook = new SpanEnrichmentHook(); + final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); final long traceId = 0x1234L; // no serial id + null variant => runtime default; object value must be JSON-stringified final Map objectValue = Collections.singletonMap("k", "val"); @@ -195,7 +207,7 @@ void runtimeDefaultMissingVariantWritesJsonObject() { final AgentSpan root = mock(AgentSpan.class); rootSpanCollection(traceId, root); - new SpanEnrichmentInterceptor().onTraceComplete(Collections.singletonList(root)); + new SpanEnrichmentInterceptor(states).onTraceComplete(Collections.singletonList(root)); // ffe_runtime_defaults is a JSON object string, NOT [object Object]/toString. verify(root) @@ -213,15 +225,12 @@ void subjectCapsAndDoLogGating() { final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); // doLog gating: addSubject only happens when doLog true (driven via the hook branch below). - final SpanEnrichmentHook hook = new SpanEnrichmentHook(); + final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); final long traceId = 0x5L; capture(hook, traceId, "f", "user-A", "on", "v", 1, false); // doLog=false => no subject - assertEquals( - 0, - SpanEnrichmentAccumulator.STATES.get(traceId).subjectCount(), - "doLog=false must not record a subject"); + assertEquals(0, states.peek(traceId).subjectCount(), "doLog=false must not record a subject"); capture(hook, traceId, "f", "user-A", "on", "v", 2, true); // doLog=true => subject recorded - assertEquals(1, SpanEnrichmentAccumulator.STATES.get(traceId).subjectCount()); + assertEquals(1, states.peek(traceId).subjectCount()); // per-subject experiment cap: 20 max for (int i = 0; i < 25; i++) { @@ -305,16 +314,23 @@ void gateOffConstructsNothingAndAccumulatesNoState() { assertFalse( hook instanceof SpanEnrichmentHook, "gate off => SpanEnrichmentHook never registered"); } - // No accumulator state created. - assertTrue(SpanEnrichmentAccumulator.STATES.isEmpty(), "gate off => no accumulator state"); + // No state store is created at all when the gate is off: the per-provider store lives only + // inside the gate-on branch, so a null interceptor means no store and no idle overhead + // (DG-005). + assertNull( + provider.spanEnrichmentInterceptor(), + "gate off => no interceptor, hence no state store and no accumulator state"); } // ---- gate-on construction + provider-close cleanup ---- @Test void gateOnConstructsHookAndInterceptorThenShutdownDisables() { - // Gate ON via the injectable override. - final Provider provider = new Provider(new Provider.Options(), null, Boolean.TRUE); + // Gate ON via the injectable override; registration succeeds (injected registrar returns true) + // so we don't mutate the global tracer (the Noop tracer rejects, which is the CR-03 path tested + // separately in secondProviderRejectedRegistrationDoesNotClearFirstProviderState). + final Provider provider = + new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); assertTrue(provider.spanEnrichmentHook() != null, "gate on => hook constructed"); assertTrue(provider.spanEnrichmentInterceptor() != null, "gate on => interceptor constructed"); assertTrue(provider.spanEnrichmentInterceptor().isEnabled()); @@ -327,28 +343,30 @@ void gateOnConstructsHookAndInterceptorThenShutdownDisables() { } assertTrue(registered, "gate on => SpanEnrichmentHook registered in getProviderHooks"); - // provider close disables the interceptor (provider-close cleanup) and drains state - SpanEnrichmentAccumulator.STATES.put(1L, new SpanEnrichmentAccumulator()); + // provider close disables the interceptor (provider-close cleanup) and drains ITS OWN state. + final SpanEnrichmentStates providerStates = provider.spanEnrichmentInterceptor().states(); + providerStates.getOrCreate(1L); + assertFalse(providerStates.isEmpty()); provider.shutdown(); assertFalse(provider.spanEnrichmentInterceptor().isEnabled(), "shutdown disables interceptor"); - assertTrue(SpanEnrichmentAccumulator.STATES.isEmpty(), "shutdown drains residual state"); + assertTrue(providerStates.isEmpty(), "shutdown drains residual state"); } // ---- error isolation: enrichment never throws ---- @Test void captureNeverThrowsOnNullInputs() { - final SpanEnrichmentHook hook = new SpanEnrichmentHook(); + final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); // null details handled in finallyAfter; capture with empty metadata + null variant hook.finallyAfter(null, null, null); - assertTrue(SpanEnrichmentAccumulator.STATES.isEmpty()); + assertTrue(states.isEmpty()); } // ---- interceptor gate-off / empty trace robustness ---- @Test void interceptorNoOpsWhenDisabledOrEmpty() { - final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(); + final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); // empty trace assertTrue(interceptor.onTraceComplete(Collections.emptyList()).isEmpty()); // disabled @@ -364,6 +382,6 @@ void interceptorNoOpsWhenDisabledOrEmpty() { @Test void interceptorPriorityIsUnique() { // Distinct from AbstractTraceInterceptor.Priority values (0,1,2,3, MAX-2, MAX-1, MAX). - assertEquals(4, new SpanEnrichmentInterceptor().priority()); + assertEquals(4, new SpanEnrichmentInterceptor(states).priority()); } } From 8491123c5d51f6782a843c2b08619cb28509bdfa Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 00:37:19 -0400 Subject: [PATCH 05/18] test(02-02): regression tests for span-enrichment lifecycle defects Adds SpanEnrichmentLifecycleRegressionTest exercising the real tracer lifecycle paths the original L0 suite bypassed (single root, no children, one provider). Each test FAILS against the pre-fix code and PASSES after the fix: CR-01 partialFlushExcludingRootPreservesPreFlushFlags / partialFlushNeverWrites- TagsOnAChildSpan: a partial flush carrying only child spans (root excluded, as dd-trace-core does) must not drain state or tag a child; the full pre+post-flush serial-id set {100,108,128,130} -> 'ZAgUAg==' must land on the root at final completion. Fail-before: tags written on the partial flush (NeverWantedButInvoked). CR-02 neverCompletingTracesDoNotLeakUnbounded / boundedStoreEvictsOldestFirst: 3x MAX_TRACES distinct never-flushed traces keep the store bounded; eviction is FIFO. Fail-before: store grew to 3x the cap (expected <=cap but was larger). CR-03 secondProviderRejectedRegistrationDoesNotClearFirstProviderState / eachProviderOwnsADistinctStateStore / hookAndInterceptorOfOneProviderShareThe- SameStore: a second provider whose registration is rejected wires no hook/interceptor and its shutdown does not clear the first provider's in-flight state. Fail-before: second provider kept a non-null interceptor (expected null). Fail-before verified by temporarily reintroducing each defect: 5/7 failed (the remaining 2 are structural-invariant checks that hold by construction). --- ...SpanEnrichmentLifecycleRegressionTest.java | 341 ++++++++++++++++++ 1 file changed, 341 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java new file mode 100644 index 00000000000..507a5aa2e82 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java @@ -0,0 +1,341 @@ +package datadog.trace.api.openfeature; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.trace.api.DDTraceId; +import datadog.trace.api.interceptor.MutableSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import dev.openfeature.sdk.EvaluationContext; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.FlagValueType; +import dev.openfeature.sdk.HookContext; +import dev.openfeature.sdk.ImmutableContext; +import dev.openfeature.sdk.ImmutableMetadata; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * GAP-CLOSURE regression suite for the three BLOCKER lifecycle defects found in the JAVA-01 code + * review (02-REVIEW-java.md). Each test reproduces a bug the original L0 suite missed because that + * suite only ever exercised a single root span, single-threaded, one provider, with the root always + * present in the flushed collection. + * + *

    + *
  • CR-01 — a PARTIAL flush (child spans only; the still-open root is excluded by + * dd-trace-core) must NOT drain the accumulator; pre-flush flags must survive onto the root + * at final completion. Verified against {@code CoreTracer.write} →{@code + * interceptCompleteTrace} firing on every flush and {@code PendingTrace.write(isPartial)} + * holding the root back. + *
  • CR-02 — traces that never reach the interceptor (dropped / Noop / never-finishing) + * must not leak accumulator entries unboundedly; the per-provider store is hard-bounded. + *
  • CR-03 — a SECOND gate-on provider whose interceptor registration is rejected + * (duplicate priority) must not wire orphan state and its {@code shutdown()} must not clear + * the FIRST provider's in-flight state. + *
+ * + *

Each test is written to FAIL against the pre-fix implementation and PASS after the fix (see + * the per-test "Fail-before" notes). + */ +class SpanEnrichmentLifecycleRegressionTest { + + // ---- helpers ---- + + private static ImmutableMetadata serialMeta(final int serialId, final boolean doLog) { + return ImmutableMetadata.builder() + .addString(SpanEnrichmentHook.METADATA_SERIAL_ID, Integer.toString(serialId)) + .addString(SpanEnrichmentHook.METADATA_DO_LOG, String.valueOf(doLog)) + .build(); + } + + private static FlagEvaluationDetails serialDetails( + final String flagKey, final int serialId, final boolean doLog) { + return FlagEvaluationDetails.builder() + .flagKey(flagKey) + .variant("on") + .value("v") + .flagMetadata(serialMeta(serialId, doLog)) + .build(); + } + + private static HookContext ctx(final String flagKey, final String targetingKey) { + final EvaluationContext ec = + targetingKey == null ? new ImmutableContext() : new ImmutableContext(targetingKey); + return HookContext.from(flagKey, FlagValueType.STRING, null, null, ec, "default"); + } + + /** A mock child span whose local root is {@code root} but which is itself NOT the root. */ + private static AgentSpan childOf(final AgentSpan root) { + final AgentSpan child = mock(AgentSpan.class); + when(child.getLocalRootSpan()).thenReturn(root); + return child; + } + + /** A mock local-root span reporting itself as its own local root with the given trace id. */ + private static AgentSpan rootSpan(final long traceId) { + final AgentSpan root = mock(AgentSpan.class); + when(root.getLocalRootSpan()).thenReturn(root); + when(root.getTraceId()).thenReturn(DDTraceId.from(traceId)); + return root; + } + + // ===================================================================================== + // CR-01: partial flush must not drop captured state or misattribute tags + // ===================================================================================== + + /** + * Models the real dd-trace-core sequence for a long-running trace: + * + *
    + *
  1. flags are evaluated (captured into the accumulator), + *
  2. a PARTIAL flush fires {@code onTraceComplete} with a fragment of CHILD spans only — the + * still-open local root is excluded (held back via {@code rootSpanWritten}), + *
  3. more flags are evaluated, + *
  4. the FINAL flush fires {@code onTraceComplete} with the root present. + *
+ * + * All serial ids — both pre- and post-partial-flush — must appear on the root's {@code + * ffe_flags_enc} tag, and nothing must be written on the partial flush. + * + *

Fail-before: the pre-fix interceptor resolved the root by reference via {@code + * getLocalRootSpan()} (reachable even when absent from the fragment) and unconditionally {@code + * remove()}d the accumulator on the FIRST (partial) flush — draining {100,108} and writing them + * onto the not-yet-finished root, so the final flush would only emit {128,130}. The assertion of + * the full {100,108,128,130} golden vector ("ZAgUAg==") on the final flush, plus "no setTag on + * the partial flush", both fail pre-fix. + */ + @Test + void partialFlushExcludingRootPreservesPreFlushFlags() { + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); + final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); + + final long traceId = 0x10A6L; + final AgentSpan root = rootSpan(traceId); + + // (1) pre-partial-flush evaluations + hook.capture(traceId, ctx("f1", "user-1"), serialDetails("f1", 100, false)); + hook.capture(traceId, ctx("f2", "user-1"), serialDetails("f2", 108, false)); + + // (2) PARTIAL flush: a fragment of children only — the open root is NOT in the collection. + final AgentSpan child1 = childOf(root); + final AgentSpan child2 = childOf(root); + final List partialFragment = Arrays.asList(child1, child2); + interceptor.onTraceComplete(partialFragment); + + // No tags may be written on a partial flush, and the accumulator must survive intact. + verify(child1, never()).setTag(anyString(), anyString()); + verify(child2, never()).setTag(anyString(), anyString()); + verify(root, never()).setTag(anyString(), anyString()); + final SpanEnrichmentAccumulator surviving = states.peek(traceId); + assertNotNull(surviving, "partial flush must NOT remove the accumulator (CR-01)"); + assertTrue( + surviving.serialIdsView().contains(100) && surviving.serialIdsView().contains(108), + "pre-flush serial ids must survive a partial flush (CR-01)"); + + // (3) more evaluations after the partial flush + hook.capture(traceId, ctx("f3", "user-1"), serialDetails("f3", 128, false)); + hook.capture(traceId, ctx("f4", "user-1"), serialDetails("f4", 130, false)); + + // (4) FINAL flush: the root is present in the fragment (alongside a late child). + final AgentSpan lateChild = childOf(root); + interceptor.onTraceComplete(Arrays.asList(lateChild, root)); + + // The full set {100,108,128,130} -> golden "ZAgUAg==" must land on the root. + verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZAgUAg=="); + assertTrue(states.isEmpty(), "state must be removed only on the final flush (CR-01)"); + } + + /** + * A partial flush whose first span reports a non-null local root that is absent from the fragment + * must be treated as "not the final write" and leave state intact — even when there are several + * children. Guards the WR-02 "never tag a non-root span" concern together with CR-01. + * + *

Fail-before: {@code findLocalRoot} returned {@code first.getLocalRootSpan()} (the + * absent root) and the interceptor wrote tags onto it / removed state on this partial fragment. + */ + @Test + void partialFlushNeverWritesTagsOnAChildSpan() { + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); + final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); + + final long traceId = 0xC0DEL; + final AgentSpan root = rootSpan(traceId); + hook.capture(traceId, ctx("f", "user-1"), serialDetails("f", 7, false)); + + final AgentSpan child = childOf(root); + interceptor.onTraceComplete(Collections.singletonList(child)); + + verify(child, never()).setTag(anyString(), anyString()); + verify(root, never()).setTag(anyString(), anyString()); + assertNotNull(states.peek(traceId), "child-only fragment must keep state (CR-01/WR-02)"); + } + + // ===================================================================================== + // CR-02: never-completing traces must not leak accumulator entries unboundedly + // ===================================================================================== + + /** + * Simulates a sustained stream of traces that each evaluate a flag but never reach the + * interceptor (dropped traces / Noop tracer / never-finishing roots). The per-provider store must + * stay bounded by {@link SpanEnrichmentStates#MAX_TRACES} rather than growing without limit. + * + *

Fail-before: state lived in an unbounded {@code static ConcurrentHashMap} with no TTL + * or cap; the only removal paths were trace-complete and shutdown. Driving 3x the cap distinct, + * never-flushed trace ids grew the map to 3x the cap — the assertion that size never exceeds the + * cap fails pre-fix (the #4844 leak class). + */ + @Test + void neverCompletingTracesDoNotLeakUnbounded() { + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); + + final int churn = SpanEnrichmentStates.MAX_TRACES * 3; + for (int i = 0; i < churn; i++) { + // Each distinct trace id accumulates once and is NEVER flushed (no interceptor call). + hook.capture(i, ctx("f", "user-" + i), serialDetails("f", i % 1000 + 1, false)); + assertTrue( + states.size() <= SpanEnrichmentStates.MAX_TRACES, + "state store must stay bounded for never-completing traces (CR-02)"); + } + assertEquals( + SpanEnrichmentStates.MAX_TRACES, + states.size(), + "store saturates at the cap, never beyond (CR-02)"); + } + + /** + * Bounding is FIFO by insertion: once the cap is reached, the oldest entries are evicted so the + * newest in-flight traces are retained. Proves eviction targets the leak (oldest, likely + * abandoned) rather than the live tail. + * + *

Fail-before: no eviction existed at all, so this behavior was absent. + */ + @Test + void boundedStoreEvictsOldestFirst() { + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + // Fill exactly to the cap with ids [0, MAX). + for (long i = 0; i < SpanEnrichmentStates.MAX_TRACES; i++) { + states.getOrCreate(i); + } + assertEquals(SpanEnrichmentStates.MAX_TRACES, states.size()); + // One more distinct id evicts the oldest (id 0) and keeps the rest + the new one. + final long overflowKey = SpanEnrichmentStates.MAX_TRACES; + states.getOrCreate(overflowKey); + assertEquals(SpanEnrichmentStates.MAX_TRACES, states.size(), "size stays at the cap"); + assertNull(states.peek(0L), "oldest entry evicted first (CR-02 FIFO)"); + assertNotNull(states.peek(overflowKey), "newest entry retained"); + assertNotNull(states.peek(1L), "second-oldest still present"); + } + + // ===================================================================================== + // CR-03: a rejected second provider must not corrupt the first provider's state + // ===================================================================================== + + /** + * Models reconfiguration: a first gate-on provider registers successfully; a SECOND gate-on + * provider's interceptor registration is REJECTED (duplicate priority 4 — what {@code + * CoreTracer.addTraceInterceptor} returns false for). The second provider must wire NO hook / + * interceptor, and its {@code shutdown()} must leave the first provider's in-flight state intact. + * + *

Fail-before: the {@code addTraceInterceptor} return value was ignored, so the second + * provider kept a non-null hook+interceptor and shared the same global {@code STATES}; its {@code + * shutdown()} called {@code STATES.clear()}, wiping the first provider's live per-trace state. + * The assertions "second provider has null hook/interceptor" and "first provider's state survives + * the second's shutdown" both fail pre-fix. + */ + @Test + void secondProviderRejectedRegistrationDoesNotClearFirstProviderState() { + // First provider: registration succeeds. + final Provider first = + new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); + assertNotNull(first.spanEnrichmentInterceptor(), "first provider registers"); + assertNotNull(first.spanEnrichmentHook(), "first provider wires a hook"); + + // First provider captures live state for an in-flight trace. + final SpanEnrichmentStates firstStates = first.spanEnrichmentInterceptor().states(); + final long liveTrace = 0xA11FEL; + first.spanEnrichmentHook().capture(liveTrace, ctx("f", "user-1"), serialDetails("f", 5, true)); + assertNotNull(firstStates.peek(liveTrace), "first provider has in-flight state"); + + // Second provider: registration is REJECTED (duplicate). It must wire nothing. + final Provider second = + new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> false); + assertNull( + second.spanEnrichmentInterceptor(), + "rejected registration => no interceptor on the second provider (CR-03)"); + assertNull( + second.spanEnrichmentHook(), + "rejected registration => no orphan hook on the second provider (CR-03)"); + + // Second provider shuts down. The first provider's live state MUST be untouched. + second.shutdown(); + assertNotNull( + firstStates.peek(liveTrace), + "second provider's shutdown must NOT clear the first provider's state (CR-03)"); + assertTrue( + first.spanEnrichmentInterceptor().isEnabled(), + "first provider's interceptor must remain enabled after the second's shutdown (CR-03)"); + + // Sanity: the first provider can still flush its own state correctly afterward. + final AgentSpan root = rootSpan(liveTrace); + first.spanEnrichmentInterceptor().onTraceComplete(Collections.singletonList(root)); + verify(root) + .setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "BQ=="); // {5} -> delta {5} -> 0x05 + assertTrue(firstStates.isEmpty(), "first provider flushes + clears its own state"); + } + + /** + * The two providers must own DISTINCT state stores — the structural guarantee behind CR-03. Even + * when both register successfully (independent tracers / test seam), one's store is never the + * other's. + * + *

Fail-before: both providers shared the single global static {@code STATES}, so this + * "distinct instance" guarantee did not hold. + */ + @Test + void eachProviderOwnsADistinctStateStore() { + final Provider a = + new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); + final Provider b = + new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); + assertNotNull(a.spanEnrichmentInterceptor()); + assertNotNull(b.spanEnrichmentInterceptor()); + assertFalse( + a.spanEnrichmentInterceptor().states() == b.spanEnrichmentInterceptor().states(), + "providers must own distinct state stores (CR-03)"); + + // Mutating a's store does not affect b's. + a.spanEnrichmentInterceptor().states().getOrCreate(1L); + assertTrue(b.spanEnrichmentInterceptor().states().isEmpty(), "stores are isolated"); + } + + /** + * Within ONE provider, the capture hook and the write interceptor must share the SAME store, so a + * capture is visible to the flush. (The cross-thread capture→flush handoff depends on this.) + */ + @Test + void hookAndInterceptorOfOneProviderShareTheSameStore() { + final Provider provider = + new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); + final long traceId = 0xBEEFL; + provider + .spanEnrichmentHook() + .capture(traceId, ctx("f", "user-1"), serialDetails("f", 9, false)); + // The interceptor's store must see what the hook captured (same instance). + assertNotNull( + provider.spanEnrichmentInterceptor().states().peek(traceId), + "hook capture must be visible in the interceptor's store (same instance)"); + } +} From 35aba868f36440648c64102c0520ccbc5a2283c2 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 17 Jun 2026 01:47:46 -0400 Subject: [PATCH 06/18] chore(feature-flagging): strip internal review labels from span-enrichment comments Remove leaked workflow labels (e.g. JAVA-01) from production comments so the upstream PR carries only behavioral context. --- .../feature-flagging/feature-flagging-api/build.gradle.kts | 2 +- .../java/datadog/trace/api/openfeature/DDEvaluator.java | 6 +++--- .../java/datadog/trace/api/openfeature/ULeb128Encoder.java | 2 +- .../java/datadog/trace/api/featureflag/ufc/v1/Split.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/build.gradle.kts b/products/feature-flagging/feature-flagging-api/build.gradle.kts index e3edd19cadc..ee5b950dc08 100644 --- a/products/feature-flagging/feature-flagging-api/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-api/build.gradle.kts @@ -45,7 +45,7 @@ dependencies { compileOnly(project(":products:feature-flagging:feature-flagging-bootstrap")) compileOnly(project(":utils:config-utils")) - // Span enrichment (JAVA-01): TraceInterceptor / GlobalTracer / AgentTracer / AgentSpan for the + // Span enrichment: TraceInterceptor / GlobalTracer / AgentTracer / AgentSpan for the // write tier + active-root-span lookup. compileOnly because the agent runtime // (feature-flagging-agent depends on :internal-api) provides these classes; the published // dd-openfeature jar must not bundle the tracer. diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index aeafbac6ba7..88fa7d4812f 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -392,9 +392,9 @@ private static ProviderEvaluation resolveVariant( .addString("flagKey", flag.key) .addString("variationType", flag.variationType.name()) .addString("allocationKey", allocation.key); - // Surface the UFC split's serial id and the allocation's doLog flag for APM span enrichment - // (JAVA-01). __dd_split_serial_id is omitted when the split carries no serial id; __dd_do_log - // is always present so the span-enrichment hook can decide whether to record the subject. + // Surface the UFC split's serial id and the allocation's doLog flag for APM span enrichment. + // __dd_split_serial_id is omitted when the split carries no serial id; __dd_do_log is always + // present so the span-enrichment hook can decide whether to record the subject. if (split.serialId != null) { metadataBuilder.addString("__dd_split_serial_id", split.serialId.toString()); } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java index 5e70c29e36d..c256a829150 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java @@ -9,7 +9,7 @@ import java.util.TreeSet; /** - * ULEB128 delta-varint + base64 codec for APM feature-flag span enrichment (JAVA-01). + * ULEB128 delta-varint + base64 codec for APM feature-flag span enrichment. * *

Ported VERBATIM from the frozen Node reference ({@code dd-trace-js#8343}). The tag names, * encoding, and golden vectors are FROZEN against that contract — backend/Trino decode and the diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/Split.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/Split.java index 73abb1ef1ec..37f6909eeea 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/Split.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/ufc/v1/Split.java @@ -9,7 +9,7 @@ public class Split { public final Map extraLogging; // Nullable Integer (not primitive int): the serialId is absent in some UFC shapes. Populated by // Moshi reflective deserialization from the UFC "serialId" JSON field. Surfaced as - // __dd_split_serial_id in eval metadata for APM span enrichment (JAVA-01). + // __dd_split_serial_id in eval metadata for APM span enrichment. public final Integer serialId; public Split( From 109eab80bc76cd01d39e3b2c75b37217e5d70ef5 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 17 Jun 2026 01:47:59 -0400 Subject: [PATCH 07/18] fix(feature-flagging): address span-enrichment code-review findings - Reconfiguration safety: replace the per-provider trace interceptor with a single process-wide delegating interceptor (SpanEnrichmentInterceptor.INSTANCE) registered once and rebound per active provider. A closing provider unbinds only if still active, so provider close/reopen no longer permanently disables enrichment via a duplicate-priority rejection, and a displaced provider's late shutdown cannot clobber the active provider's in-flight state. - Runtime defaults: recursively unwrap OpenFeature Value (structure/list/scalar) to its native form before JSON serialization so ffe_runtime_defaults matches the frozen Node contract instead of emitting Value.toString(). - 128-bit trace ids: key per-trace state by the full trace id hex string rather than DDTraceId.toLong(), so two traces sharing low-order 64 bits no longer merge enrichment state. - Gate-off inertness: precompute the immutable provider-hook list once in the constructor so getProviderHooks() (called per evaluation) allocates nothing. - Add regression tests: reconfiguration, displaced-provider late shutdown, 128-bit low-bit collision, Value structure/list/scalar serialization, and gate-off zero-allocation. --- .../trace/api/openfeature/Provider.java | 98 +++--- .../SpanEnrichmentAccumulator.java | 106 ++++-- .../api/openfeature/SpanEnrichmentHook.java | 24 +- .../SpanEnrichmentInterceptor.java | 129 +++++--- .../api/openfeature/SpanEnrichmentStates.java | 43 +-- .../openfeature/SpanEnrichmentHookTest.java | 191 ++++++++--- ...SpanEnrichmentLifecycleRegressionTest.java | 305 ++++++++++-------- 7 files changed, 580 insertions(+), 316 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index 5fe38851635..158e3e76842 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -47,9 +47,12 @@ public class Provider extends EventProvider implements Metadata { new AtomicReference<>(InitializationState.NOT_STARTED); private final FlagEvalMetrics flagEvalMetrics; private final FlagEvalHook flagEvalHook; - // Span enrichment (JAVA-01): both are null unless the gate is on (DG-005 — no idle overhead). + // Span enrichment: null unless the gate is on, so the feature has no idle overhead when off. private final SpanEnrichmentHook spanEnrichmentHook; - private final SpanEnrichmentInterceptor spanEnrichmentInterceptor; + private final SpanEnrichmentStates spanEnrichmentStates; + // Precomputed hook list returned by getProviderHooks() on every evaluation. Immutable and built + // once so gate-off evaluation allocates nothing on this hot path. + private final List providerHooks; public Provider() { this(DEFAULT_OPTIONS, null); @@ -65,10 +68,10 @@ public Provider(final Options options) { /** * Registers a {@link SpanEnrichmentInterceptor} with the running tracer, returning {@code true} - * when it was added and {@code false} when the tracer rejected it (e.g. a duplicate-priority - * interceptor from an earlier provider is already registered). Injectable so tests can drive the - * success and rejected-registration (reconfiguration) paths deterministically without mutating - * the global tracer (mirrors the {@code spanEnrichmentEnabledOverride} seam). + * when it was added and {@code false} when the tracer rejected it (e.g. the interceptor is + * already registered, or the global tracer is the no-op placeholder). Injectable so tests can + * drive registration deterministically without mutating the global tracer (mirrors the {@code + * spanEnrichmentEnabledOverride} seam). */ interface TraceInterceptorRegistrar { boolean register(SpanEnrichmentInterceptor interceptor); @@ -113,49 +116,46 @@ interface TraceInterceptorRegistrar { this.flagEvalMetrics = metrics; this.flagEvalHook = hook; - // Gate-gated span enrichment: construct + register ONLY when the gate is on. When off, nothing - // is constructed and no interceptor/state exists (DG-005 zero-idle-overhead negative control). + // Span enrichment is wired ONLY when the gate is on. When off, no hook/state is constructed and + // there is no idle per-evaluation or per-span overhead. final boolean spanEnrichmentEnabled = spanEnrichmentEnabledOverride != null ? spanEnrichmentEnabledOverride : isSpanEnrichmentEnabled(); SpanEnrichmentHook seHook = null; - SpanEnrichmentInterceptor seInterceptor = null; + SpanEnrichmentStates seStates = null; if (spanEnrichmentEnabled) { try { - // Per-provider state store (NOT a global static): owned by the interceptor, shared with the - // hook, so this provider's shutdown can never clear another provider's state (CR-03), and a - // never-completing trace cannot leak unboundedly (CR-02, bounded inside the store). - final SpanEnrichmentStates seStates = new SpanEnrichmentStates(); - final SpanEnrichmentInterceptor candidate = new SpanEnrichmentInterceptor(seStates); - // HONOR the registration result (CR-03): the tracer rejects an interceptor whose priority - // is - // already taken (e.g. a SECOND gate-on provider after reconfiguration) and returns false. - // If - // we ignored it, this provider would still wire a hook into shared state yet never have its - // interceptor invoked, and its shutdown() would clear the FIRST provider's live state. So - // we - // only wire the hook+interceptor when registration actually succeeded. - if (registrar.register(candidate)) { - seInterceptor = candidate; - seHook = new SpanEnrichmentHook(seStates); - } else { - // Duplicate registration (active interceptor already owns priority 4). Leave hook + - // interceptor null so this provider neither accumulates orphan state nor clears the - // active provider's state on shutdown. Its own seStates is unreferenced and GC'd. - log.warn( - "Span enrichment interceptor already registered (duplicate provider); " - + "skipping duplicate registration to avoid corrupting active provider state"); - } + // Per-provider state store, shared with this provider's capture hook. The single, + // process-wide interceptor is registered once (reconfiguration-safe) and rebound to this + // provider's store. A later gate-on provider rebinds it to its own store; this provider's + // shutdown only unbinds if it is still the active provider, so reconfiguration never + // permanently disables enrichment and providers never clobber each other's live state. + seStates = new SpanEnrichmentStates(); + SpanEnrichmentInterceptor.ensureRegistered(registrar); + SpanEnrichmentInterceptor.INSTANCE.bind(seStates); + seHook = new SpanEnrichmentHook(seStates); } catch (LinkageError | Exception e) { // Tracer classes absent (e.g. API-only classpath): degrade to no span enrichment. log.warn("Span enrichment unavailable — tracer classes not on classpath", e); seHook = null; - seInterceptor = null; + seStates = null; } } this.spanEnrichmentHook = seHook; - this.spanEnrichmentInterceptor = seInterceptor; + this.spanEnrichmentStates = seStates; + + // Precompute the immutable hook list once so getProviderHooks() (called on every evaluation) + // allocates nothing, including when the gate is off. + final List hooks = new ArrayList<>(2); + if (flagEvalHook != null) { + hooks.add(flagEvalHook); + } + if (seHook != null) { + hooks.add(seHook); + } + this.providerHooks = + hooks.isEmpty() ? Collections.emptyList() : Collections.unmodifiableList(hooks); } private static boolean isSpanEnrichmentEnabled() { @@ -274,14 +274,7 @@ private Evaluator buildEvaluator() throws Exception { @Override public List getProviderHooks() { - final List hooks = new ArrayList<>(2); - if (flagEvalHook != null) { - hooks.add(flagEvalHook); - } - if (spanEnrichmentHook != null) { - hooks.add(spanEnrichmentHook); - } - return hooks.isEmpty() ? Collections.emptyList() : hooks; + return providerHooks; } @Override @@ -289,14 +282,13 @@ public void shutdown() { if (flagEvalMetrics != null) { flagEvalMetrics.shutdown(); } - // Provider-close cleanup for span enrichment (Pitfall 3): the tracer has no interceptor-removal - // API, so disable the interceptor (it then no-ops + drains its OWN residual state). Because the - // state store is instance-owned, this clears only this provider's state and can never wipe an - // active second provider's in-flight state (CR-03). When this provider's registration was - // rejected as a duplicate, spanEnrichmentInterceptor is null here, so shutdown is a no-op and - // the active provider's state is untouched. - if (spanEnrichmentInterceptor != null) { - spanEnrichmentInterceptor.disable(); + // Provider-close cleanup for span enrichment: the tracer has no interceptor-removal API, so we + // unbind this provider's store from the process-wide interceptor (which clears the store and + // makes the interceptor inert until a new provider rebinds it). The unbind is a no-op if a + // newer provider has already rebound the interceptor, so we never wipe another provider's + // in-flight state. + if (spanEnrichmentStates != null) { + SpanEnrichmentInterceptor.INSTANCE.unbind(spanEnrichmentStates); } if (evaluator != null) { evaluator.shutdown(); @@ -308,8 +300,8 @@ SpanEnrichmentHook spanEnrichmentHook() { return spanEnrichmentHook; } - SpanEnrichmentInterceptor spanEnrichmentInterceptor() { - return spanEnrichmentInterceptor; + SpanEnrichmentStates spanEnrichmentStates() { + return spanEnrichmentStates; } @Override diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java index df37c42bf37..76265a89c50 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java @@ -1,25 +1,27 @@ package datadog.trace.api.openfeature; +import dev.openfeature.sdk.Structure; +import dev.openfeature.sdk.Value; +import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.TreeSet; /** - * Per-local-root-span accumulator for APM feature-flag span enrichment (JAVA-01). + * Per-local-root-span accumulator for APM feature-flag span enrichment. * *

Holds the serial ids, hashed subjects, and runtime defaults captured during flag evaluation * for a single local trace fragment. The limits, dedupe semantics, truncation, and output tag * shapes are FROZEN against the Node reference ({@code dd-trace-js#8343}) — see {@link * ULeb128Encoder}. * - *

Instances are created lazily and held in a per-provider {@link SpanEnrichmentStates} store, - * keyed by the local-root span's trace id. The capture hook ({@link SpanEnrichmentHook}) writes; - * the write interceptor ({@link SpanEnrichmentInterceptor}) reads and clears. When the - * span-enrichment gate is off, no store and no accumulator are ever created, so there is no idle - * per-span overhead (DG-005). + *

Instances are created lazily and held in a {@link SpanEnrichmentStates} store, keyed by the + * local-root span's full trace id. The capture hook ({@link SpanEnrichmentHook}) writes; the write + * interceptor ({@link SpanEnrichmentInterceptor}) reads and clears. When the span-enrichment gate + * is off, no store and no accumulator are ever created, so there is no idle per-span overhead. * - *

Output tag shapes (Pattern F): + *

Output tag shapes: * *

    *
  • {@code ffe_flags_enc} — a bare base64 string (delta-varint of the serial ids) @@ -139,22 +141,87 @@ synchronized Map toSpanTags() { // ---- helpers (visible for tests) ---- /** - * Mirrors the Node {@code (typeof object && !null) ? JSON.stringify(value) : String(value)} rule: - * structured values (maps, lists) are JSON-stringified, scalars use their string form, null - * becomes the JSON literal {@code null}. + * Mirrors the Node {@code (typeof value === 'object' && value !== null) ? JSON.stringify(value) : + * String(value)} rule: structured values (objects, arrays) are JSON-stringified; scalars use + * their string form; {@code null} becomes the bare {@code null}. + * + *

    On the real OpenFeature object-evaluation path the runtime-default arrives wrapped in a + * {@link Value}; we unwrap it to its native representation first so a structured default + * serializes to JSON (matching Node's {@code JSON.stringify} of the equivalent JS object) instead + * of {@code Value.toString()} (which is {@code "Value(innerObject=...)"}). The scalar cases + * ({@code String}/{@code Boolean}/number) collapse to the same string form Node produces. */ static String stringifyDefault(final Object value) { - if (value == null) { + final Object unwrapped = value instanceof Value ? unwrapValue((Value) value) : value; + if (unwrapped == null) { return "null"; } - if (value instanceof Map || value instanceof Iterable || value.getClass().isArray()) { - return toJsonValue(value); + if (unwrapped instanceof Map + || unwrapped instanceof Iterable + || unwrapped.getClass().isArray()) { + return toJsonValue(unwrapped); + } + if (unwrapped instanceof CharSequence || unwrapped instanceof Character) { + return unwrapped.toString(); + } + // Numbers / booleans / Instant — their string form matches what Node's String(value) emits for + // these scalar cases. + return String.valueOf(unwrapped); + } + + /** + * Recursively unwraps an OpenFeature {@link Value} into its native Java representation: + * structures become {@code Map}, lists become {@code List}, and scalars + * become their boxed value (or {@code null}). Nested {@link Value}s are unwrapped at every level + * so a structure containing further structures/lists serializes correctly. + */ + private static Object unwrapValue(final Value value) { + if (value == null || value.isNull()) { + return null; + } + if (value.isStructure()) { + final Structure structure = value.asStructure(); + final Map map = new LinkedHashMap<>(); + if (structure != null) { + for (final String key : structure.keySet()) { + map.put(key, unwrapValue(structure.getValue(key))); + } + } + return map; + } + if (value.isList()) { + final java.util.List list = value.asList(); + final java.util.List out = new java.util.ArrayList<>(list == null ? 0 : list.size()); + if (list != null) { + for (final Value element : list) { + out.add(unwrapValue(element)); + } + } + return out; + } + if (value.isBoolean()) { + return value.asBoolean(); + } + if (value.isString()) { + return value.asString(); + } + if (value.isNumber()) { + // Preserve integral vs fractional so the rendered JSON number matches Node. + final Double d = value.asDouble(); + if (d != null && d == Math.rint(d) && !Double.isInfinite(d)) { + final Integer i = value.asInteger(); + if (i != null) { + return i; + } + } + return d; } - if (value instanceof CharSequence || value instanceof Character) { - return value.toString(); + final Instant instant = value.asInstant(); + if (instant != null) { + return instant.toString(); } - // Numbers / booleans — their string form matches JSON for these scalar cases. - return String.valueOf(value); + // Unknown shape: fall back to the wrapped object's own representation. + return value.asObject(); } /** UTF-8-safe truncation: never split a surrogate pair at the {@code maxChars} boundary. */ @@ -194,7 +261,10 @@ private static String toJsonValue(final Object value) { } @SuppressWarnings("unchecked") - private static void appendJsonValue(final StringBuilder sb, final Object value) { + private static void appendJsonValue(final StringBuilder sb, final Object rawValue) { + // Unwrap any OpenFeature Value to its native representation first so nested structures/lists + // serialize as JSON rather than via Value.toString(). + final Object value = rawValue instanceof Value ? unwrapValue((Value) rawValue) : rawValue; if (value == null) { sb.append("null"); } else if (value instanceof Map) { diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java index 42d1c4eef61..9d469efee2b 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java @@ -11,17 +11,17 @@ /** * OpenFeature {@code finally} hook that captures feature-flag evaluation metadata into - * per-local-root span state for APM span enrichment (JAVA-01). This is the CAPTURE half of the + * per-local-root span state for APM span enrichment. This is the CAPTURE half of the * capture-vs-write split — the WRITE half is {@link SpanEnrichmentInterceptor}, which flushes the * accumulated tags onto the local root span when the trace completes. * *

    Mirrors {@link FlagEvalHook}: registered via {@link Provider#getProviderHooks()} (only when * the span-enrichment gate is on) and reading {@code details.getFlagMetadata()}. It resolves the - * active local-root span via {@link AgentTracer#activeSpan()} and keys the per-provider {@link - * SpanEnrichmentStates} store (shared with this provider's {@link SpanEnrichmentInterceptor}) by - * that root's trace id, so the interceptor running later on the write thread can recover the same - * state from the completed span collection. The store is owned by the interceptor (not a global - * static), so a second provider can never clear this one's state (CR-03). + * active local-root span via {@link AgentTracer#activeSpan()} and keys the {@link + * SpanEnrichmentStates} store (shared with the {@link SpanEnrichmentInterceptor}) by that root's + * full trace id (hex), so the interceptor running later on the write thread can recover the same + * state from the completed span collection. The full hex key avoids merging two distinct 128-bit + * traces that happen to share their low-order 64 bits. * *

    Capture branch (frozen Node reference): * @@ -31,7 +31,7 @@ *

  • else variant missing (runtime default) → addDefault(flagKey, value) * * - *

    All work is wrapped in try/catch — enrichment must NEVER break flag evaluation (Pattern D). + *

    All work is wrapped in try/catch — enrichment must NEVER break flag evaluation. */ class SpanEnrichmentHook implements Hook { @@ -56,9 +56,7 @@ interface RootSpanResolver { }; private final RootSpanResolver rootSpanResolver; - // Per-provider state store, shared with this provider's interceptor (NOT a global static). The - // hook writes here; the interceptor reads + removes. Owning the store on the interceptor is what - // makes one provider's shutdown unable to clear another's in-flight state (CR-03). + // State store shared with the interceptor. The hook writes here; the interceptor reads + removes. private final SpanEnrichmentStates states; SpanEnrichmentHook(final SpanEnrichmentStates states) { @@ -83,7 +81,9 @@ public void finallyAfter( if (root == null || root.getTraceId() == null) { return; // no active span → nothing to enrich } - final long traceKey = root.getTraceId().toLong(); + // Key by the full trace id (hex), not toLong(): the low-order 64 bits alone would merge two + // distinct 128-bit traces that share their low bits. + final String traceKey = root.getTraceId().toHexString(); capture(traceKey, ctx, details); } catch (final Throwable t) { // Never let span enrichment break flag evaluation. @@ -95,7 +95,7 @@ public void finallyAfter( * private so it can be driven deterministically in tests without stubbing the static tracer. */ void capture( - final long traceKey, + final String traceKey, final HookContext ctx, final FlagEvaluationDetails details) { final ImmutableMetadata metadata = details.getFlagMetadata(); diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java index 24aefc14ca9..8e6bc9d45a1 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java @@ -6,17 +6,33 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import java.util.Collection; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; /** - * {@link TraceInterceptor} that writes the accumulated {@code ffe_*} tags onto the local root span - * when a trace actually completes (JAVA-01). This is the WRITE half of the - * capture-vs-write split — the CAPTURE half is {@link SpanEnrichmentHook}, which fills this - * interceptor's per-provider {@link SpanEnrichmentStates} store during flag evaluation. + * Process-wide {@link TraceInterceptor} that writes the accumulated {@code ffe_*} tags onto the + * local root span when a trace actually completes. This is the WRITE half of the + * capture-vs-write split — the CAPTURE half is {@link SpanEnrichmentHook}, which fills the active + * {@link SpanEnrichmentStates} store during flag evaluation. * - *

    Partial-flush correctness (CR-01)

    + *

    Reconfiguration safety

    + * + *

    The tracer holds interceptors in an add-only, priority-keyed set with no public removal API: a + * given priority can be occupied by exactly one interceptor for the life of the tracer. If each + * provider registered its own interceptor at the same priority, the first registration would win + * forever; after that provider closed, every later gate-on provider would be rejected as a + * duplicate and enrichment would be permanently disabled. + * + *

    To survive provider close/reopen we therefore register a single, long-lived delegating + * interceptor ({@link #INSTANCE}) exactly once, and {@linkplain #bind(SpanEnrichmentStates) rebind} + * it to whichever provider is currently active. A provider {@linkplain + * #unbind(SpanEnrichmentStates) unbinds} on close. When no provider is bound the interceptor is + * inert; a fresh provider rebinds it and enrichment resumes — no second registration is ever + * attempted. + * + *

    Partial-flush correctness

    * *

    {@code dd-trace-core} runs {@code onTraceComplete} on every flush, not only on final - * trace completion: {@code CoreTracer.write(SpanList)} →{@code interceptCompleteTrace(...)} fires + * trace completion: {@code CoreTracer.write(SpanList)} → {@code interceptCompleteTrace(...)} fires * for both partial flushes ({@code PendingTrace.partialFlush()} → {@code write(true)}) and the * final write ({@code write(false)}). A partial flush deliberately excludes the * still-open local root — {@code PendingTrace} holds the root back ({@code rootSpanWritten}) @@ -30,14 +46,14 @@ * the accumulator and write tags onto a not-yet-finished root, silently dropping all pre-flush * enrichment — exactly the long-running-trace data-loss bug. * - *

    State ownership (CR-02 / CR-03)

    + *

    State ownership

    * - *

    State lives in a per-interceptor (per-provider) {@link SpanEnrichmentStates} store, not a - * global static. {@link #disable()} clears only this instance's store, so one provider's shutdown - * can never wipe another's in-flight state (CR-03). The store is hard-bounded, so a trace that - * never reaches this interceptor cannot leak unboundedly (CR-02). + *

    State lives in the per-provider {@link SpanEnrichmentStates} store that is currently bound. + * {@link #unbind(SpanEnrichmentStates)} clears only the store it unbinds, so one provider's close + * can never wipe another's in-flight state. The store is hard-bounded, so a trace that never + * reaches this interceptor cannot leak unboundedly. * - *

    All work is wrapped in try/catch — enrichment must NEVER break trace finish (Pattern D). + *

    All work is wrapped in try/catch — enrichment must NEVER break trace finish. */ final class SpanEnrichmentInterceptor implements TraceInterceptor { @@ -48,53 +64,85 @@ final class SpanEnrichmentInterceptor implements TraceInterceptor { */ static final int PRIORITY = 4; - // The tracer holds interceptors in an add-only sorted set keyed by priority with no public - // removal API. On provider shutdown we cannot un-register, so we disable this instance instead: - // a disabled interceptor no-ops and drains its own residual state (Pitfall 3 — provider-close - // cleanup). Because state is instance-owned, disabling never touches another provider's state. - private volatile boolean enabled = true; + /** The single, long-lived interceptor registered with the tracer (reconfiguration safety). */ + static final SpanEnrichmentInterceptor INSTANCE = new SpanEnrichmentInterceptor(); + + // Whether INSTANCE has been accepted by a real tracer. Registration can legitimately fail when + // the global tracer is still the no-op (not yet installed); in that case we leave this false so a + // later provider retries. Once true we never re-attempt (the interceptor is in for good). + private final AtomicBoolean registered = new AtomicBoolean(false); - // Per-provider state store, shared with this provider's hook. Owned here so cleanup is scoped. - private final SpanEnrichmentStates states; + // The store of the currently-active provider. null when no gate-on provider is bound, in which + // case the interceptor is inert. Volatile: the eval/bind threads write, the trace-write thread + // reads. + private volatile SpanEnrichmentStates activeStates; - SpanEnrichmentInterceptor(final SpanEnrichmentStates states) { - this.states = states; + private SpanEnrichmentInterceptor() {} + + /** + * Idempotently registers {@link #INSTANCE} with the tracer via {@code registrar}. Safe to call + * from every gate-on provider: the first successful registration wins and subsequent calls are + * no-ops. If registration fails because the tracer is not yet installed, a later call retries. + */ + static void ensureRegistered(final Provider.TraceInterceptorRegistrar registrar) { + if (INSTANCE.registered.get()) { + return; + } + if (registrar.register(INSTANCE)) { + INSTANCE.registered.set(true); + } + } + + /** Binds the active store to {@code states}, displacing any previously-bound provider. */ + void bind(final SpanEnrichmentStates states) { + this.activeStates = states; } - /** The state store shared with this interceptor's capture hook. */ - SpanEnrichmentStates states() { - return states; + /** + * Unbinds {@code states} if it is still the active store and clears it (cleanup on provider + * close). If a newer provider has already rebound the interceptor, this is a no-op so the new + * provider's in-flight state is left untouched. + */ + void unbind(final SpanEnrichmentStates states) { + if (this.activeStates == states) { + this.activeStates = null; + } + if (states != null) { + states.clear(); + } } - /** Disables the interceptor and clears its OWN accumulated state (provider-close cleanup). */ - void disable() { - enabled = false; - states.clear(); + /** The currently-bound store, or {@code null} when the interceptor is inert. */ + SpanEnrichmentStates activeStates() { + return activeStates; } - boolean isEnabled() { - return enabled; + boolean isRegistered() { + return registered.get(); } @Override public Collection onTraceComplete( final Collection trace) { try { - if (!enabled || trace == null || trace.isEmpty()) { + final SpanEnrichmentStates states = this.activeStates; + if (states == null || trace == null || trace.isEmpty()) { return trace; } // Resolve the local root for this fragment, then require that the root is actually PRESENT in - // this collection. A partial flush excludes the still-open root (CR-01), so its absence means - // "not the final write" — keep the accumulator and bail. + // this collection. A partial flush excludes the still-open root, so its absence means "not + // the + // final write" — keep the accumulator and bail. final MutableSpan localRoot = findLocalRootInFragment(trace); if (!(localRoot instanceof AgentSpan)) { return trace; // partial flush, or no resolvable in-fragment root: keep state untouched } final DDTraceId traceId = ((AgentSpan) localRoot).getTraceId(); if (traceId == null) { - return trace; // no trace id (e.g. Noop span) — cannot key state; keep it (WR-01) + return trace; // no trace id (e.g. Noop span) — cannot key state; keep it } - final long traceKey = traceId.toLong(); + // Key by the full trace id (hex) to match the capture-side keying. + final String traceKey = traceId.toHexString(); final SpanEnrichmentAccumulator state = states.remove(traceKey); if (state == null || !state.hasData()) { return trace; @@ -114,12 +162,11 @@ public Collection onTraceComplete( /** * Resolves the local root span for this fragment and returns it ONLY if it is actually present in * the fragment by reference identity. Returns {@code null} when the root is not in the collection - * (a partial flush excludes the still-open root — CR-01) or when no root can be safely - * identified. + * (a partial flush excludes the still-open root) or when no root can be safely identified. * - *

    We never guess: a non-root span is never returned (WR-02). If the first span reports a - * non-null local root, we accept it only after confirming that exact object is in the fragment; - * otherwise we look for a span that is provably its own local root and present. + *

    We never guess: a non-root span is never returned. If the first span reports a non-null + * local root, we accept it only after confirming that exact object is in the fragment; otherwise + * we look for a span that is provably its own local root and present. */ private static MutableSpan findLocalRootInFragment( final Collection trace) { @@ -136,7 +183,7 @@ private static MutableSpan findLocalRootInFragment( return null; // root excluded from this fragment → partial flush, do not flush/remove } // Local root unknown for the first span: only accept a span that is provably its own local root - // and present here. Never fall back to an arbitrary span (WR-02). + // and present here. Never fall back to an arbitrary span. for (final MutableSpan span : trace) { if (span.getLocalRootSpan() == span) { return span; diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java index 8bd81433c77..df120d8a6f2 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java @@ -8,21 +8,24 @@ /** * Bounded, instance-owned store of per-trace {@link SpanEnrichmentAccumulator} state, keyed by the - * local-root span's trace id ({@code DDTraceId.toLong()}). + * local-root span's full trace id (the lower-case zero-padded hex string). * - *

    This replaces the previous global {@code static} map (review IN-03 / CR-02 / CR-03). Each - * {@link SpanEnrichmentInterceptor} owns exactly one instance and shares it with the {@link - * SpanEnrichmentHook} for the same provider, so: + *

    The full hex key matters for 128-bit trace ids: keying by {@code DDTraceId.toLong()} (the + * low-order 64 bits only) would merge two distinct 128-bit traces that share their low bits into a + * single accumulator, cross-contaminating enrichment between unrelated traces. The full hex string + * is unique across all 128 bits and is cached on the id, so it is cheap to obtain. + * + *

    Each {@link SpanEnrichmentHook}/{@link SpanEnrichmentInterceptor} pair shares one store + * instance, so: * *

      - *
    • CR-02 (unbounded leak): the store is hard-capped at {@link #MAX_TRACES}. A trace - * that never reaches the interceptor (dropped, Noop tracer, never-finishing root) can no - * longer leak unboundedly — once the cap is reached, the oldest in-flight entry is - * evicted (FIFO by insertion order) so the map size is strictly bounded regardless of trace - * completion. This is the exact #4844 leak class. - *
    • CR-03 (reconfiguration corruption): because the store is per-interceptor rather than - * a shared static, one provider's {@code shutdown()} clears only its own state and can never - * wipe another (still-active) provider's in-flight entries. + *
    • Unbounded leak: the store is hard-capped at {@link #MAX_TRACES}. A trace that never + * reaches the interceptor (dropped, Noop tracer, never-finishing root) can no longer leak + * unboundedly — once the cap is reached, the oldest in-flight entry is evicted (FIFO + * by insertion order) so the map size is strictly bounded regardless of trace completion. + *
    • Isolation: because the store is instance-owned rather than a shared static, one + * provider's cleanup clears only its own state and can never wipe another (still-active) + * provider's in-flight entries. *
    * *

    Bounded eviction is intentionally lossy under pathological pressure: dropping the oldest @@ -40,14 +43,14 @@ final class SpanEnrichmentStates { /** * Hard cap on the number of concurrently-tracked traces. Sized well above any realistic count of * simultaneously in-flight traces with active flag evaluations on a single JVM, so the live path - * never evicts; the cap exists purely to bound a leak of never-completing traces (CR-02). + * never evicts; the cap exists purely to bound a leak of never-completing traces. */ static final int MAX_TRACES = 4096; // Insertion-ordered so the eldest entry is the natural eviction victim (FIFO). accessOrder=false // (the default) — we evict by age of creation, not by recency of use, so a long-running but // never-completing trace cannot pin the map by being repeatedly touched. - private final LinkedHashMap states = new LinkedHashMap<>(); + private final LinkedHashMap states = new LinkedHashMap<>(); // One-shot guard so a sustained leak logs once at WARN rather than on every eviction. private boolean evictionWarned = false; @@ -55,9 +58,9 @@ final class SpanEnrichmentStates { /** * Returns the accumulator for {@code traceKey}, creating (and inserting) it if absent. When * insertion would exceed {@link #MAX_TRACES}, the eldest entry is evicted first so the store - * stays bounded (CR-02). + * stays bounded. */ - synchronized SpanEnrichmentAccumulator getOrCreate(final long traceKey) { + synchronized SpanEnrichmentAccumulator getOrCreate(final String traceKey) { SpanEnrichmentAccumulator existing = states.get(traceKey); if (existing != null) { return existing; @@ -71,11 +74,11 @@ synchronized SpanEnrichmentAccumulator getOrCreate(final long traceKey) { } /** Removes and returns the accumulator for {@code traceKey}, or {@code null} if absent. */ - synchronized SpanEnrichmentAccumulator remove(final long traceKey) { + synchronized SpanEnrichmentAccumulator remove(final String traceKey) { return states.remove(traceKey); } - /** Clears all tracked state (provider-close cleanup). Scoped to this instance only (CR-03). */ + /** Clears all tracked state (cleanup on provider close / unbind). */ synchronized void clear() { states.clear(); } @@ -90,12 +93,12 @@ synchronized boolean isEmpty() { // ---- test-only accessor ---- - synchronized SpanEnrichmentAccumulator peek(final long traceKey) { + synchronized SpanEnrichmentAccumulator peek(final String traceKey) { return states.get(traceKey); } private void evictEldest() { - final Iterator> it = states.entrySet().iterator(); + final Iterator> it = states.entrySet().iterator(); if (it.hasNext()) { it.next(); it.remove(); diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java index 4b166fe2abf..b3d2a197d39 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java @@ -20,7 +20,11 @@ import dev.openfeature.sdk.HookContext; import dev.openfeature.sdk.ImmutableContext; import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.ImmutableStructure; +import dev.openfeature.sdk.Value; +import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.SortedSet; @@ -30,16 +34,17 @@ import org.junit.jupiter.api.Test; /** - * L0 unit suite for APM feature-flag span enrichment (JAVA-01). + * Unit suite for APM feature-flag span enrichment. * - *

    Covers the seven required VALIDATION.md cases plus the explicit max-200 case and the codec + *

    Covers the seven required validation cases plus the explicit max-200 case and the codec * golden-vector round-trip. The contract (encoding, limits, tag shapes) is FROZEN against the Node * reference ({@code dd-trace-js#8343}). */ class SpanEnrichmentHookTest { - // Per-test instance-owned state store (replaces the former global static). The hook and the - // interceptor under test share this one, mirroring how a single Provider wires them (CR-03). + // Instance-owned state store shared by the hook and interceptor under test, mirroring how a + // single + // Provider wires them. private SpanEnrichmentStates states; @BeforeEach @@ -50,10 +55,17 @@ void freshState() { @AfterEach void clearState() { states.clear(); + // The interceptor is a process-wide singleton; leave it inert for the next test. + SpanEnrichmentInterceptor.INSTANCE.unbind(SpanEnrichmentInterceptor.INSTANCE.activeStates()); } // ---- helpers ---- + /** The store key the production code derives from a trace id (full hex). */ + private static String key(final long traceId) { + return DDTraceId.from(traceId).toHexString(); + } + private static FlagEvaluationDetails details( final String flagKey, final String variant, @@ -86,10 +98,10 @@ private static HookContext ctx(final String flagKey, final String target "default"); } - /** Drives the capture branch directly (no static tracer) for a fixed trace key. */ + /** Drives the capture branch directly (no static tracer) for a fixed trace id. */ private static void capture( final SpanEnrichmentHook hook, - final long traceKey, + final long traceId, final String flagKey, final String targetingKey, final String variant, @@ -97,7 +109,7 @@ private static void capture( final Integer serialId, final boolean doLog) { hook.capture( - traceKey, + key(traceId), ctx(flagKey, targetingKey), details(flagKey, variant, value, metadata(serialId, doLog))); } @@ -105,7 +117,7 @@ private static void capture( /** * Configures a mock root span to report itself as its own local root with the given trace id. The * returned span, passed in a singleton collection, models a FINAL flush (the root is present in - * the fragment), so the interceptor flushes + removes state (CR-01 final-write path). + * the fragment), so the interceptor flushes + removes state. */ private static AgentSpan rootSpanCollection(final long traceId, final AgentSpan rootSpan) { when(rootSpan.getLocalRootSpan()).thenReturn(rootSpan); @@ -113,6 +125,12 @@ private static AgentSpan rootSpanCollection(final long traceId, final AgentSpan return rootSpan; } + /** Binds a fresh interceptor to the given store, models a final flush, and returns it. */ + private static SpanEnrichmentInterceptor boundInterceptor(final SpanEnrichmentStates states) { + SpanEnrichmentInterceptor.INSTANCE.bind(states); + return SpanEnrichmentInterceptor.INSTANCE; + } + // ---- 1. codec golden-vector round-trip ---- @Test @@ -158,7 +176,7 @@ void finallyAfterResolvesRootViaResolverAndAccumulates() { ctx("flag", "user-1"), details("flag", "on", "v", metadata(42, true)), Collections.emptyMap()); - final SpanEnrichmentAccumulator state = states.peek(0x77L); + final SpanEnrichmentAccumulator state = states.peek(key(0x77L)); assertTrue(state != null && state.serialIdsView().contains(42)); assertEquals(1, state.subjectCount(), "doLog=true + targeting key => subject recorded"); } @@ -178,9 +196,7 @@ void finishedRootFlushesFlagsEncTag() { final AgentSpan root = mock(AgentSpan.class); rootSpanCollection(traceId, root); - final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); - - interceptor.onTraceComplete(Collections.singletonList(root)); + boundInterceptor(states).onTraceComplete(Collections.singletonList(root)); verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZAgUAg=="); // state cleared after flush @@ -196,7 +212,7 @@ void runtimeDefaultMissingVariantWritesJsonObject() { // no serial id + null variant => runtime default; object value must be JSON-stringified final Map objectValue = Collections.singletonMap("k", "val"); hook.capture( - traceId, + key(traceId), ctx("obj-flag", "user-1"), FlagEvaluationDetails.builder() .flagKey("obj-flag") @@ -207,7 +223,7 @@ void runtimeDefaultMissingVariantWritesJsonObject() { final AgentSpan root = mock(AgentSpan.class); rootSpanCollection(traceId, root); - new SpanEnrichmentInterceptor(states).onTraceComplete(Collections.singletonList(root)); + boundInterceptor(states).onTraceComplete(Collections.singletonList(root)); // ffe_runtime_defaults is a JSON object string, NOT [object Object]/toString. verify(root) @@ -218,6 +234,80 @@ void runtimeDefaultMissingVariantWritesJsonObject() { verify(root, never()).setTag(eq(SpanEnrichmentAccumulator.TAG_FLAGS_ENC), anyString()); } + // ---- 4b. real OpenFeature object path: a Value structure default must serialize to JSON ---- + + /** + * The real object-evaluation path hands the runtime default in as a {@code + * dev.openfeature.sdk.Value}, not a raw {@code Map}. The accumulator must unwrap the {@code + * Value} and emit JSON (matching Node's {@code JSON.stringify}), never {@code Value.toString()} + * (which is {@code "Value(innerObject=...)"}). + */ + @Test + void runtimeDefaultStructureValueSerializesAsJsonNotToString() { + final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); + final long traceId = 0x2468L; + + // Single-key structure so the exact-string assertion does not depend on the OpenFeature SDK's + // internal key ordering. (The multi-key / nesting behaviour is covered by the direct + // stringifyDefault assertions below.) + final Map inner = new LinkedHashMap<>(); + inner.put("enabled", new Value(true)); + final Value structureDefault = new Value(new ImmutableStructure(inner)); + + hook.capture( + key(traceId), + ctx("struct-flag", "user-1"), + FlagEvaluationDetails.builder() + .flagKey("struct-flag") + .variant(null) + .value(structureDefault) + .flagMetadata(ImmutableMetadata.builder().build()) + .build()); + + final AgentSpan root = mock(AgentSpan.class); + rootSpanCollection(traceId, root); + boundInterceptor(states).onTraceComplete(Collections.singletonList(root)); + + // {"struct-flag":"{\"enabled\":true}"} — note: NO "Value(innerObject=...)". + verify(root) + .setTag( + SpanEnrichmentAccumulator.TAG_RUNTIME_DEFAULTS, + "{\"struct-flag\":\"{\\\"enabled\\\":true}\"}"); + } + + /** + * Direct stringify of a structured {@code Value}: asserts the value is JSON (objects + nested + * scalars), not {@code Value.toString()}. The structure is single-key to keep ordering + * deterministic across OpenFeature SDK versions. + */ + @Test + void stringifyDefaultUnwrapsValueStructureToJson() { + final Map inner = new LinkedHashMap<>(); + inner.put("count", new Value(42)); + final Value structureDefault = new Value(new ImmutableStructure(inner)); + assertEquals("{\"count\":42}", SpanEnrichmentAccumulator.stringifyDefault(structureDefault)); + } + + /** + * A list-valued {@code Value} default serializes to a JSON array, with nested Values unwrapped. + */ + @Test + void runtimeDefaultListValueSerializesAsJsonArray() { + final Value listDefault = + new Value(Arrays.asList(new Value("a"), new Value(2), new Value(true))); + // direct stringify assertion (exact bytes) + assertEquals("[\"a\",2,true]", SpanEnrichmentAccumulator.stringifyDefault(listDefault)); + } + + /** Scalar Values collapse to the same string form Node's String(value) produces. */ + @Test + void scalarValueDefaultsMatchNodeStringForm() { + assertEquals("hello", SpanEnrichmentAccumulator.stringifyDefault(new Value("hello"))); + assertEquals("true", SpanEnrichmentAccumulator.stringifyDefault(new Value(true))); + assertEquals("7", SpanEnrichmentAccumulator.stringifyDefault(new Value(7))); + assertEquals("null", SpanEnrichmentAccumulator.stringifyDefault(new Value())); + } + // ---- 5. per-subject cap (10 subjects / 20 experiments / doLog gating) ---- @Test @@ -228,9 +318,10 @@ void subjectCapsAndDoLogGating() { final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); final long traceId = 0x5L; capture(hook, traceId, "f", "user-A", "on", "v", 1, false); // doLog=false => no subject - assertEquals(0, states.peek(traceId).subjectCount(), "doLog=false must not record a subject"); + assertEquals( + 0, states.peek(key(traceId)).subjectCount(), "doLog=false must not record a subject"); capture(hook, traceId, "f", "user-A", "on", "v", 2, true); // doLog=true => subject recorded - assertEquals(1, states.peek(traceId).subjectCount()); + assertEquals(1, states.peek(key(traceId)).subjectCount()); // per-subject experiment cap: 20 max for (int i = 0; i < 25; i++) { @@ -298,42 +389,54 @@ void objectDefaultJsonAndTruncation() { assertEquals(1, acc.defaultCount()); } - // ---- gate-off negative control (no ffe_*, no hook/interceptor, no state) (DG-005) ---- + // ---- gate-off negative control (no ffe_*, no hook, no state) ---- @Test void gateOffConstructsNothingAndAccumulatesNoState() { // Gate OFF via the injectable override (no static config mocking). final Provider provider = new Provider(new Provider.Options(), null, Boolean.FALSE); - // No hook, no interceptor constructed (DG-005 zero-idle-overhead). + // No hook, no state store constructed (zero idle overhead). assertNull(provider.spanEnrichmentHook(), "gate off => no span-enrichment hook"); - assertNull(provider.spanEnrichmentInterceptor(), "gate off => no span-enrichment interceptor"); + assertNull(provider.spanEnrichmentStates(), "gate off => no span-enrichment state store"); // getProviderHooks must not contain a SpanEnrichmentHook. final List hooks = provider.getProviderHooks(); for (final Hook hook : hooks) { assertFalse( hook instanceof SpanEnrichmentHook, "gate off => SpanEnrichmentHook never registered"); } - // No state store is created at all when the gate is off: the per-provider store lives only - // inside the gate-on branch, so a null interceptor means no store and no idle overhead - // (DG-005). - assertNull( - provider.spanEnrichmentInterceptor(), - "gate off => no interceptor, hence no state store and no accumulator state"); + } + + /** + * getProviderHooks() is called on every evaluation; it must return the SAME precomputed list + * (allocating nothing) regardless of gate state. + */ + @Test + void getProviderHooksReturnsSameInstanceEachCall() { + final Provider gateOff = new Provider(new Provider.Options(), null, Boolean.FALSE); + assertTrue( + gateOff.getProviderHooks() == gateOff.getProviderHooks(), + "gate off => getProviderHooks allocates nothing (same instance)"); + + final Provider gateOn = + new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); + assertTrue( + gateOn.getProviderHooks() == gateOn.getProviderHooks(), + "gate on => getProviderHooks allocates nothing (same instance)"); } // ---- gate-on construction + provider-close cleanup ---- @Test - void gateOnConstructsHookAndInterceptorThenShutdownDisables() { - // Gate ON via the injectable override; registration succeeds (injected registrar returns true) - // so we don't mutate the global tracer (the Noop tracer rejects, which is the CR-03 path tested - // separately in secondProviderRejectedRegistrationDoesNotClearFirstProviderState). + void gateOnConstructsHookAndStateThenShutdownUnbinds() { final Provider provider = new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); assertTrue(provider.spanEnrichmentHook() != null, "gate on => hook constructed"); - assertTrue(provider.spanEnrichmentInterceptor() != null, "gate on => interceptor constructed"); - assertTrue(provider.spanEnrichmentInterceptor().isEnabled()); + assertTrue(provider.spanEnrichmentStates() != null, "gate on => state store constructed"); + // the process-wide interceptor is bound to this provider's store + assertTrue( + SpanEnrichmentInterceptor.INSTANCE.activeStates() == provider.spanEnrichmentStates(), + "gate on => interceptor bound to this provider's store"); // hook registered in provider hooks boolean registered = false; for (final Hook hook : provider.getProviderHooks()) { @@ -343,12 +446,13 @@ void gateOnConstructsHookAndInterceptorThenShutdownDisables() { } assertTrue(registered, "gate on => SpanEnrichmentHook registered in getProviderHooks"); - // provider close disables the interceptor (provider-close cleanup) and drains ITS OWN state. - final SpanEnrichmentStates providerStates = provider.spanEnrichmentInterceptor().states(); - providerStates.getOrCreate(1L); + // provider close unbinds + drains ITS OWN state. + final SpanEnrichmentStates providerStates = provider.spanEnrichmentStates(); + providerStates.getOrCreate(key(1L)); assertFalse(providerStates.isEmpty()); provider.shutdown(); - assertFalse(provider.spanEnrichmentInterceptor().isEnabled(), "shutdown disables interceptor"); + assertNull( + SpanEnrichmentInterceptor.INSTANCE.activeStates(), "shutdown unbinds the active store"); assertTrue(providerStates.isEmpty(), "shutdown drains residual state"); } @@ -362,18 +466,19 @@ void captureNeverThrowsOnNullInputs() { assertTrue(states.isEmpty()); } - // ---- interceptor gate-off / empty trace robustness ---- + // ---- interceptor inert when unbound / empty trace robustness ---- @Test - void interceptorNoOpsWhenDisabledOrEmpty() { - final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); + void interceptorNoOpsWhenUnboundOrEmpty() { // empty trace - assertTrue(interceptor.onTraceComplete(Collections.emptyList()).isEmpty()); - // disabled - interceptor.disable(); + SpanEnrichmentInterceptor.INSTANCE.bind(states); + assertTrue( + SpanEnrichmentInterceptor.INSTANCE.onTraceComplete(Collections.emptyList()).isEmpty()); + // unbound (inert) + SpanEnrichmentInterceptor.INSTANCE.unbind(states); final AgentSpan root = mock(AgentSpan.class); final List trace = Collections.singletonList(root); - interceptor.onTraceComplete(trace); + SpanEnrichmentInterceptor.INSTANCE.onTraceComplete(trace); verify(root, never()).setTag(anyString(), anyString()); } @@ -382,6 +487,6 @@ void interceptorNoOpsWhenDisabledOrEmpty() { @Test void interceptorPriorityIsUnique() { // Distinct from AbstractTraceInterceptor.Priority values (0,1,2,3, MAX-2, MAX-1, MAX). - assertEquals(4, new SpanEnrichmentInterceptor(states).priority()); + assertEquals(4, SpanEnrichmentInterceptor.INSTANCE.priority()); } } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java index 507a5aa2e82..7e4d00b5875 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java @@ -1,7 +1,6 @@ package datadog.trace.api.openfeature; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -11,6 +10,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import datadog.trace.api.DD128bTraceId; import datadog.trace.api.DDTraceId; import datadog.trace.api.interceptor.MutableSpan; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -23,34 +23,41 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; /** - * GAP-CLOSURE regression suite for the three BLOCKER lifecycle defects found in the JAVA-01 code - * review (02-REVIEW-java.md). Each test reproduces a bug the original L0 suite missed because that - * suite only ever exercised a single root span, single-threaded, one provider, with the root always - * present in the flushed collection. + * Regression suite for the per-root-span lifecycle and reconfiguration behaviour of span + * enrichment. Each test exercises a scenario the single-root, single-thread, single-provider happy + * path does not cover. * *
      - *
    • CR-01 — a PARTIAL flush (child spans only; the still-open root is excluded by + *
    • partial flush — a fragment of child spans only (the still-open root is excluded by * dd-trace-core) must NOT drain the accumulator; pre-flush flags must survive onto the root - * at final completion. Verified against {@code CoreTracer.write} →{@code - * interceptCompleteTrace} firing on every flush and {@code PendingTrace.write(isPartial)} - * holding the root back. - *
    • CR-02 — traces that never reach the interceptor (dropped / Noop / never-finishing) - * must not leak accumulator entries unboundedly; the per-provider store is hard-bounded. - *
    • CR-03 — a SECOND gate-on provider whose interceptor registration is rejected - * (duplicate priority) must not wire orphan state and its {@code shutdown()} must not clear - * the FIRST provider's in-flight state. + * at final completion. + *
    • unbounded leak — traces that never reach the interceptor (dropped / Noop / never-finishing) + * must not leak accumulator entries unboundedly; the store is hard-bounded. + *
    • reconfiguration — after a provider closes, a new gate-on provider must still enrich (the + * process-wide interceptor rebinds), and a closing provider must never clobber a newer + * provider's in-flight state. + *
    • 128-bit trace ids — two distinct 128-bit traces that share their low-order 64 bits must not + * merge enrichment state. *
    - * - *

    Each test is written to FAIL against the pre-fix implementation and PASS after the fix (see - * the per-test "Fail-before" notes). */ class SpanEnrichmentLifecycleRegressionTest { + @AfterEach + void resetInterceptor() { + // The interceptor is a process-wide singleton; leave it inert for the next test. + SpanEnrichmentInterceptor.INSTANCE.unbind(SpanEnrichmentInterceptor.INSTANCE.activeStates()); + } + // ---- helpers ---- + private static String key(final long traceId) { + return DDTraceId.from(traceId).toHexString(); + } + private static ImmutableMetadata serialMeta(final int serialId, final boolean doLog) { return ImmutableMetadata.builder() .addString(SpanEnrichmentHook.METADATA_SERIAL_ID, Integer.toString(serialId)) @@ -82,15 +89,19 @@ private static AgentSpan childOf(final AgentSpan root) { } /** A mock local-root span reporting itself as its own local root with the given trace id. */ - private static AgentSpan rootSpan(final long traceId) { + private static AgentSpan rootSpan(final DDTraceId traceId) { final AgentSpan root = mock(AgentSpan.class); when(root.getLocalRootSpan()).thenReturn(root); - when(root.getTraceId()).thenReturn(DDTraceId.from(traceId)); + when(root.getTraceId()).thenReturn(traceId); return root; } + private static AgentSpan rootSpan(final long traceId) { + return rootSpan(DDTraceId.from(traceId)); + } + // ===================================================================================== - // CR-01: partial flush must not drop captured state or misattribute tags + // partial flush must not drop captured state or misattribute tags // ===================================================================================== /** @@ -106,26 +117,20 @@ private static AgentSpan rootSpan(final long traceId) { * * All serial ids — both pre- and post-partial-flush — must appear on the root's {@code * ffe_flags_enc} tag, and nothing must be written on the partial flush. - * - *

    Fail-before: the pre-fix interceptor resolved the root by reference via {@code - * getLocalRootSpan()} (reachable even when absent from the fragment) and unconditionally {@code - * remove()}d the accumulator on the FIRST (partial) flush — draining {100,108} and writing them - * onto the not-yet-finished root, so the final flush would only emit {128,130}. The assertion of - * the full {100,108,128,130} golden vector ("ZAgUAg==") on the final flush, plus "no setTag on - * the partial flush", both fail pre-fix. */ @Test void partialFlushExcludingRootPreservesPreFlushFlags() { final SpanEnrichmentStates states = new SpanEnrichmentStates(); final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); + final SpanEnrichmentInterceptor interceptor = SpanEnrichmentInterceptor.INSTANCE; + interceptor.bind(states); final long traceId = 0x10A6L; final AgentSpan root = rootSpan(traceId); // (1) pre-partial-flush evaluations - hook.capture(traceId, ctx("f1", "user-1"), serialDetails("f1", 100, false)); - hook.capture(traceId, ctx("f2", "user-1"), serialDetails("f2", 108, false)); + hook.capture(key(traceId), ctx("f1", "user-1"), serialDetails("f1", 100, false)); + hook.capture(key(traceId), ctx("f2", "user-1"), serialDetails("f2", 108, false)); // (2) PARTIAL flush: a fragment of children only — the open root is NOT in the collection. final AgentSpan child1 = childOf(root); @@ -137,15 +142,15 @@ void partialFlushExcludingRootPreservesPreFlushFlags() { verify(child1, never()).setTag(anyString(), anyString()); verify(child2, never()).setTag(anyString(), anyString()); verify(root, never()).setTag(anyString(), anyString()); - final SpanEnrichmentAccumulator surviving = states.peek(traceId); - assertNotNull(surviving, "partial flush must NOT remove the accumulator (CR-01)"); + final SpanEnrichmentAccumulator surviving = states.peek(key(traceId)); + assertNotNull(surviving, "partial flush must NOT remove the accumulator"); assertTrue( surviving.serialIdsView().contains(100) && surviving.serialIdsView().contains(108), - "pre-flush serial ids must survive a partial flush (CR-01)"); + "pre-flush serial ids must survive a partial flush"); // (3) more evaluations after the partial flush - hook.capture(traceId, ctx("f3", "user-1"), serialDetails("f3", 128, false)); - hook.capture(traceId, ctx("f4", "user-1"), serialDetails("f4", 130, false)); + hook.capture(key(traceId), ctx("f3", "user-1"), serialDetails("f3", 128, false)); + hook.capture(key(traceId), ctx("f4", "user-1"), serialDetails("f4", 130, false)); // (4) FINAL flush: the root is present in the fragment (alongside a late child). final AgentSpan lateChild = childOf(root); @@ -153,48 +158,41 @@ void partialFlushExcludingRootPreservesPreFlushFlags() { // The full set {100,108,128,130} -> golden "ZAgUAg==" must land on the root. verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZAgUAg=="); - assertTrue(states.isEmpty(), "state must be removed only on the final flush (CR-01)"); + assertTrue(states.isEmpty(), "state must be removed only on the final flush"); } /** * A partial flush whose first span reports a non-null local root that is absent from the fragment * must be treated as "not the final write" and leave state intact — even when there are several - * children. Guards the WR-02 "never tag a non-root span" concern together with CR-01. - * - *

    Fail-before: {@code findLocalRoot} returned {@code first.getLocalRootSpan()} (the - * absent root) and the interceptor wrote tags onto it / removed state on this partial fragment. + * children. Guards the "never tag a non-root span" concern. */ @Test void partialFlushNeverWritesTagsOnAChildSpan() { final SpanEnrichmentStates states = new SpanEnrichmentStates(); final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); + final SpanEnrichmentInterceptor interceptor = SpanEnrichmentInterceptor.INSTANCE; + interceptor.bind(states); final long traceId = 0xC0DEL; final AgentSpan root = rootSpan(traceId); - hook.capture(traceId, ctx("f", "user-1"), serialDetails("f", 7, false)); + hook.capture(key(traceId), ctx("f", "user-1"), serialDetails("f", 7, false)); final AgentSpan child = childOf(root); interceptor.onTraceComplete(Collections.singletonList(child)); verify(child, never()).setTag(anyString(), anyString()); verify(root, never()).setTag(anyString(), anyString()); - assertNotNull(states.peek(traceId), "child-only fragment must keep state (CR-01/WR-02)"); + assertNotNull(states.peek(key(traceId)), "child-only fragment must keep state"); } // ===================================================================================== - // CR-02: never-completing traces must not leak accumulator entries unboundedly + // never-completing traces must not leak accumulator entries unboundedly // ===================================================================================== /** * Simulates a sustained stream of traces that each evaluate a flag but never reach the - * interceptor (dropped traces / Noop tracer / never-finishing roots). The per-provider store must - * stay bounded by {@link SpanEnrichmentStates#MAX_TRACES} rather than growing without limit. - * - *

    Fail-before: state lived in an unbounded {@code static ConcurrentHashMap} with no TTL - * or cap; the only removal paths were trace-complete and shutdown. Driving 3x the cap distinct, - * never-flushed trace ids grew the map to 3x the cap — the assertion that size never exceeds the - * cap fails pre-fix (the #4844 leak class). + * interceptor (dropped traces / Noop tracer / never-finishing roots). The store must stay bounded + * by {@link SpanEnrichmentStates#MAX_TRACES} rather than growing without limit. */ @Test void neverCompletingTracesDoNotLeakUnbounded() { @@ -204,126 +202,135 @@ void neverCompletingTracesDoNotLeakUnbounded() { final int churn = SpanEnrichmentStates.MAX_TRACES * 3; for (int i = 0; i < churn; i++) { // Each distinct trace id accumulates once and is NEVER flushed (no interceptor call). - hook.capture(i, ctx("f", "user-" + i), serialDetails("f", i % 1000 + 1, false)); + hook.capture(key(i), ctx("f", "user-" + i), serialDetails("f", i % 1000 + 1, false)); assertTrue( states.size() <= SpanEnrichmentStates.MAX_TRACES, - "state store must stay bounded for never-completing traces (CR-02)"); + "state store must stay bounded for never-completing traces"); } assertEquals( - SpanEnrichmentStates.MAX_TRACES, - states.size(), - "store saturates at the cap, never beyond (CR-02)"); + SpanEnrichmentStates.MAX_TRACES, states.size(), "store saturates at the cap, never beyond"); } /** * Bounding is FIFO by insertion: once the cap is reached, the oldest entries are evicted so the - * newest in-flight traces are retained. Proves eviction targets the leak (oldest, likely - * abandoned) rather than the live tail. - * - *

    Fail-before: no eviction existed at all, so this behavior was absent. + * newest in-flight traces are retained. */ @Test void boundedStoreEvictsOldestFirst() { final SpanEnrichmentStates states = new SpanEnrichmentStates(); // Fill exactly to the cap with ids [0, MAX). for (long i = 0; i < SpanEnrichmentStates.MAX_TRACES; i++) { - states.getOrCreate(i); + states.getOrCreate(key(i)); } assertEquals(SpanEnrichmentStates.MAX_TRACES, states.size()); // One more distinct id evicts the oldest (id 0) and keeps the rest + the new one. - final long overflowKey = SpanEnrichmentStates.MAX_TRACES; - states.getOrCreate(overflowKey); + final long overflow = SpanEnrichmentStates.MAX_TRACES; + states.getOrCreate(key(overflow)); assertEquals(SpanEnrichmentStates.MAX_TRACES, states.size(), "size stays at the cap"); - assertNull(states.peek(0L), "oldest entry evicted first (CR-02 FIFO)"); - assertNotNull(states.peek(overflowKey), "newest entry retained"); - assertNotNull(states.peek(1L), "second-oldest still present"); + assertNull(states.peek(key(0L)), "oldest entry evicted first (FIFO)"); + assertNotNull(states.peek(key(overflow)), "newest entry retained"); + assertNotNull(states.peek(key(1L)), "second-oldest still present"); } // ===================================================================================== - // CR-03: a rejected second provider must not corrupt the first provider's state + // reconfiguration: a closing provider must not permanently disable enrichment, nor + // clobber a newer provider's state // ===================================================================================== /** - * Models reconfiguration: a first gate-on provider registers successfully; a SECOND gate-on - * provider's interceptor registration is REJECTED (duplicate priority 4 — what {@code - * CoreTracer.addTraceInterceptor} returns false for). The second provider must wire NO hook / - * interceptor, and its {@code shutdown()} must leave the first provider's in-flight state intact. - * - *

    Fail-before: the {@code addTraceInterceptor} return value was ignored, so the second - * provider kept a non-null hook+interceptor and shared the same global {@code STATES}; its {@code - * shutdown()} called {@code STATES.clear()}, wiping the first provider's live per-trace state. - * The assertions "second provider has null hook/interceptor" and "first provider's state survives - * the second's shutdown" both fail pre-fix. + * The core reconfiguration regression: a first gate-on provider shuts down, then a second gate-on + * provider is created. Enrichment must still work for the second provider. The process-wide + * interceptor is registered once and rebound, so the second provider is NOT rejected as a + * duplicate (which would permanently disable enrichment). */ @Test - void secondProviderRejectedRegistrationDoesNotClearFirstProviderState() { - // First provider: registration succeeds. + void newProviderAfterShutdownStillEnriches() { + // First provider registers and binds. final Provider first = new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); - assertNotNull(first.spanEnrichmentInterceptor(), "first provider registers"); - assertNotNull(first.spanEnrichmentHook(), "first provider wires a hook"); + assertNotNull(first.spanEnrichmentStates(), "first provider wires a store"); + first.shutdown(); + assertNull( + SpanEnrichmentInterceptor.INSTANCE.activeStates(), + "first provider's shutdown leaves the interceptor inert"); - // First provider captures live state for an in-flight trace. - final SpanEnrichmentStates firstStates = first.spanEnrichmentInterceptor().states(); - final long liveTrace = 0xA11FEL; - first.spanEnrichmentHook().capture(liveTrace, ctx("f", "user-1"), serialDetails("f", 5, true)); - assertNotNull(firstStates.peek(liveTrace), "first provider has in-flight state"); + // Second provider is created AFTER the first closed. With the old per-provider interceptor + // model + // this registration would be rejected as a duplicate and enrichment would be permanently off. + final Provider second = + new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); + final SpanEnrichmentStates secondStates = second.spanEnrichmentStates(); + assertNotNull(secondStates, "second provider wires a store"); + assertTrue( + SpanEnrichmentInterceptor.INSTANCE.activeStates() == secondStates, + "the interceptor must rebind to the second provider's store"); - // Second provider: registration is REJECTED (duplicate). It must wire nothing. + // End-to-end: the second provider captures and the interceptor flushes onto the root. + final long traceId = 0xBEE5L; + second + .spanEnrichmentHook() + .capture(key(traceId), ctx("f", "user-1"), serialDetails("f", 5, false)); + final AgentSpan root = rootSpan(traceId); + SpanEnrichmentInterceptor.INSTANCE.onTraceComplete(Collections.singletonList(root)); + verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "BQ=="); // {5} -> 0x05 + } + + /** + * Overlapping reconfiguration: a second provider rebinds the interceptor while the first is still + * "open"; when the FIRST provider later shuts down, it must NOT clear the second provider's + * in-flight state (its unbind is a no-op because it is no longer the active provider). + */ + @Test + void lateShutdownOfDisplacedProviderDoesNotClobberActiveProvider() { + final Provider first = + new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); final Provider second = - new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> false); - assertNull( - second.spanEnrichmentInterceptor(), - "rejected registration => no interceptor on the second provider (CR-03)"); - assertNull( - second.spanEnrichmentHook(), - "rejected registration => no orphan hook on the second provider (CR-03)"); + new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); + final SpanEnrichmentStates secondStates = second.spanEnrichmentStates(); - // Second provider shuts down. The first provider's live state MUST be untouched. - second.shutdown(); - assertNotNull( - firstStates.peek(liveTrace), - "second provider's shutdown must NOT clear the first provider's state (CR-03)"); + // Second provider is now the active one and has live state. + final long liveTrace = 0xA11FEL; + second + .spanEnrichmentHook() + .capture(key(liveTrace), ctx("f", "user-1"), serialDetails("f", 9, false)); + assertNotNull(secondStates.peek(key(liveTrace)), "second provider has in-flight state"); + assertTrue(SpanEnrichmentInterceptor.INSTANCE.activeStates() == secondStates); + + // The DISPLACED first provider shuts down late. The active (second) provider must be untouched. + first.shutdown(); assertTrue( - first.spanEnrichmentInterceptor().isEnabled(), - "first provider's interceptor must remain enabled after the second's shutdown (CR-03)"); + SpanEnrichmentInterceptor.INSTANCE.activeStates() == secondStates, + "late shutdown of a displaced provider must not unbind the active provider"); + assertNotNull( + secondStates.peek(key(liveTrace)), + "late shutdown of a displaced provider must not clear the active provider's state"); - // Sanity: the first provider can still flush its own state correctly afterward. + // Sanity: the second provider still flushes correctly. final AgentSpan root = rootSpan(liveTrace); - first.spanEnrichmentInterceptor().onTraceComplete(Collections.singletonList(root)); - verify(root) - .setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "BQ=="); // {5} -> delta {5} -> 0x05 - assertTrue(firstStates.isEmpty(), "first provider flushes + clears its own state"); + SpanEnrichmentInterceptor.INSTANCE.onTraceComplete(Collections.singletonList(root)); + verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "CQ=="); // {9} -> 0x09 } - /** - * The two providers must own DISTINCT state stores — the structural guarantee behind CR-03. Even - * when both register successfully (independent tracers / test seam), one's store is never the - * other's. - * - *

    Fail-before: both providers shared the single global static {@code STATES}, so this - * "distinct instance" guarantee did not hold. - */ + /** Each provider owns a DISTINCT state store, so they never share mutable state. */ @Test void eachProviderOwnsADistinctStateStore() { final Provider a = new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); final Provider b = new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); - assertNotNull(a.spanEnrichmentInterceptor()); - assertNotNull(b.spanEnrichmentInterceptor()); - assertFalse( - a.spanEnrichmentInterceptor().states() == b.spanEnrichmentInterceptor().states(), - "providers must own distinct state stores (CR-03)"); - - // Mutating a's store does not affect b's. - a.spanEnrichmentInterceptor().states().getOrCreate(1L); - assertTrue(b.spanEnrichmentInterceptor().states().isEmpty(), "stores are isolated"); + assertNotNull(a.spanEnrichmentStates()); + assertNotNull(b.spanEnrichmentStates()); + assertTrue( + a.spanEnrichmentStates() != b.spanEnrichmentStates(), + "providers must own distinct state stores"); + + a.spanEnrichmentStates().getOrCreate(key(1L)); + assertTrue(b.spanEnrichmentStates().isEmpty(), "stores are isolated"); } /** - * Within ONE provider, the capture hook and the write interceptor must share the SAME store, so a - * capture is visible to the flush. (The cross-thread capture→flush handoff depends on this.) + * Within ONE provider, the capture hook and the bound interceptor must share the SAME store, so a + * capture is visible to the flush. */ @Test void hookAndInterceptorOfOneProviderShareTheSameStore() { @@ -332,10 +339,50 @@ void hookAndInterceptorOfOneProviderShareTheSameStore() { final long traceId = 0xBEEFL; provider .spanEnrichmentHook() - .capture(traceId, ctx("f", "user-1"), serialDetails("f", 9, false)); - // The interceptor's store must see what the hook captured (same instance). + .capture(key(traceId), ctx("f", "user-1"), serialDetails("f", 9, false)); assertNotNull( - provider.spanEnrichmentInterceptor().states().peek(traceId), - "hook capture must be visible in the interceptor's store (same instance)"); + SpanEnrichmentInterceptor.INSTANCE.activeStates().peek(key(traceId)), + "hook capture must be visible in the bound store (same instance)"); + } + + // ===================================================================================== + // 128-bit trace ids that share their low-order 64 bits must not merge + // ===================================================================================== + + /** + * Two distinct 128-bit trace ids whose low-order 64 bits are identical (so {@code toLong()} + * collides) must keep SEPARATE accumulators. Keying by {@code toLong()} would merge them; keying + * by the full hex string keeps them apart. + */ + @Test + void distinct128BitTraceIdsSharingLowBitsDoNotMerge() { + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); + final SpanEnrichmentInterceptor interceptor = SpanEnrichmentInterceptor.INSTANCE; + interceptor.bind(states); + + // Same low 64 bits (0xABCD), different high 64 bits — toLong() is identical for both. + final long lowBits = 0xABCDL; + final DDTraceId idA = DD128bTraceId.from(0x1111L, lowBits); + final DDTraceId idB = DD128bTraceId.from(0x2222L, lowBits); + assertEquals(idA.toLong(), idB.toLong(), "precondition: low 64 bits collide"); + assertTrue(!idA.toHexString().equals(idB.toHexString()), "precondition: full ids differ"); + + // Capture distinct serial ids under each full trace id. + hook.capture(idA.toHexString(), ctx("fa", "user-A"), serialDetails("fa", 100, false)); + hook.capture(idB.toHexString(), ctx("fb", "user-B"), serialDetails("fb", 130, false)); + + // They must NOT have merged into one accumulator. + assertEquals(2, states.size(), "distinct 128-bit traces must not share an accumulator"); + + // Flush trace A: only {100} (not {100,130}). + final AgentSpan rootA = rootSpan(idA); + interceptor.onTraceComplete(Collections.singletonList(rootA)); + verify(rootA).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZA=="); // {100} -> 0x64 + + // Flush trace B: only {130}. + final AgentSpan rootB = rootSpan(idB); + interceptor.onTraceComplete(Collections.singletonList(rootB)); + verify(rootB).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ggE="); // {130} -> 0x82 0x01 } } From 7b4da18b4aa53554d6b1d6d15cf9a2837bc370ae Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 2 Jul 2026 16:26:37 +0300 Subject: [PATCH 08/18] Adjustments + clean up; add FeatureFlaggingConfig.SPAN_ENRICHMENT_ENABLED; add Startup INFO log; clean tech debt --- .../trace/api/openfeature/Provider.java | 31 +++++++++++++------ .../SpanEnrichmentInterceptor.java | 4 --- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index 0b57686f155..5cc32e7553b 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -3,7 +3,8 @@ import static java.util.concurrent.TimeUnit.SECONDS; import datadog.trace.api.GlobalTracer; -import datadog.trace.config.inversion.ConfigHelper; +import datadog.trace.api.config.FeatureFlaggingConfig; +import datadog.trace.bootstrap.config.provider.ConfigProvider; import de.thetaphi.forbiddenapis.SuppressForbidden; import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.EvaluationContext; @@ -33,12 +34,14 @@ public class Provider extends EventProvider implements Metadata { private static final String EVALUATOR_IMPL = "datadog.trace.api.openfeature.DDEvaluator"; /** - * Environment variable form of {@link - * datadog.trace.api.config.FeatureFlaggingConfig#SPAN_ENRICHMENT_ENABLED}. Distinct from the - * provider-enabled gate; OFF by default (experimental opt-in). + * Canonical config key for the span-enrichment gate ({@link + * FeatureFlaggingConfig#SPAN_ENRICHMENT_ENABLED}). Read through {@link ConfigProvider} so the + * full precedence applies — system property, env var ({@code + * DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED}), and stable config — exactly like + * the sibling provider-enabled gate. Distinct from the provider-enabled gate; OFF by default + * (experimental opt-in). */ - static final String SPAN_ENRICHMENT_ENABLED_ENV = - "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED"; + static final String SPAN_ENRICHMENT_ENABLED_KEY = FeatureFlaggingConfig.SPAN_ENRICHMENT_ENABLED; private static final Options DEFAULT_OPTIONS = new Options().initTimeout(30, SECONDS); private volatile Evaluator evaluator; @@ -79,7 +82,7 @@ interface TraceInterceptorRegistrar { /** * @param spanEnrichmentEnabledOverride when non-null, forces the span-enrichment gate (test - * seam); when null, the gate is read from {@link #SPAN_ENRICHMENT_ENABLED_ENV}. + * seam); when null, the gate is read from {@link #SPAN_ENRICHMENT_ENABLED_KEY}. */ Provider( final Options options, @@ -155,12 +158,22 @@ interface TraceInterceptorRegistrar { } this.providerHooks = hooks.isEmpty() ? Collections.emptyList() : Collections.unmodifiableList(hooks); + + // Announce the span-enrichment state at startup (matches the reference implementation). + // Reflects effective wiring: "enabled" only when the hook was actually constructed (gate on AND + // tracer classes present), otherwise "disabled". + if (spanEnrichmentHook != null) { + log.info("{} span enrichment enabled", METADATA); + } else { + log.info("{} span enrichment disabled", METADATA); + } } private static boolean isSpanEnrichmentEnabled() { try { - final String value = ConfigHelper.env(SPAN_ENRICHMENT_ENABLED_ENV); - return "true".equalsIgnoreCase(value) || "1".equals(value); + // Full config precedence (system property > stable config > env) via ConfigProvider, matching + // the sibling provider-enabled gate. "1"/"true" (any case) map to true; default false. + return ConfigProvider.getInstance().getBoolean(SPAN_ENRICHMENT_ENABLED_KEY, false); } catch (final Throwable t) { return false; // never let config reading break provider construction } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java index 8e6bc9d45a1..b9b65c839ab 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java @@ -117,10 +117,6 @@ SpanEnrichmentStates activeStates() { return activeStates; } - boolean isRegistered() { - return registered.get(); - } - @Override public Collection onTraceComplete( final Collection trace) { From d29a8b38b2077185b74bab1b3c6e04822f74729d Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 7 Jul 2026 16:08:38 +0300 Subject: [PATCH 09/18] Rework FFE span-enrichment state store to weak-keyed-by-span map --- .../api/openfeature/SpanEnrichmentHook.java | 25 +-- .../SpanEnrichmentInterceptor.java | 14 +- .../api/openfeature/SpanEnrichmentStates.java | 101 +++------ .../openfeature/SpanEnrichmentHookTest.java | 68 +++--- ...SpanEnrichmentLifecycleRegressionTest.java | 201 ++++++++++-------- 5 files changed, 180 insertions(+), 229 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java index 9d469efee2b..6fe84b6d1e6 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java @@ -18,10 +18,9 @@ *

    Mirrors {@link FlagEvalHook}: registered via {@link Provider#getProviderHooks()} (only when * the span-enrichment gate is on) and reading {@code details.getFlagMetadata()}. It resolves the * active local-root span via {@link AgentTracer#activeSpan()} and keys the {@link - * SpanEnrichmentStates} store (shared with the {@link SpanEnrichmentInterceptor}) by that root's - * full trace id (hex), so the interceptor running later on the write thread can recover the same - * state from the completed span collection. The full hex key avoids merging two distinct 128-bit - * traces that happen to share their low-order 64 bits. + * SpanEnrichmentStates} store (shared with the {@link SpanEnrichmentInterceptor}) by that root span + * object, so the interceptor running later on the write thread can recover the same state from the + * completed span collection (the local root is a single stable object for the trace). * *

    Capture branch (frozen Node reference): * @@ -78,24 +77,22 @@ public void finallyAfter( } try { final AgentSpan root = rootSpanResolver.activeLocalRoot(); - if (root == null || root.getTraceId() == null) { + if (root == null) { return; // no active span → nothing to enrich } - // Key by the full trace id (hex), not toLong(): the low-order 64 bits alone would merge two - // distinct 128-bit traces that share their low bits. - final String traceKey = root.getTraceId().toHexString(); - capture(traceKey, ctx, details); + capture(root, ctx, details); } catch (final Throwable t) { // Never let span enrichment break flag evaluation. } } /** - * Applies the frozen Node capture branch against the state keyed by {@code traceKey}. Package - * private so it can be driven deterministically in tests without stubbing the static tracer. + * Applies the frozen Node capture branch against the state keyed by the local-root {@code root} + * span. Package private so it can be driven deterministically in tests without stubbing the + * static tracer. */ void capture( - final String traceKey, + final AgentSpan root, final HookContext ctx, final FlagEvaluationDetails details) { final ImmutableMetadata metadata = details.getFlagMetadata(); @@ -111,14 +108,14 @@ void capture( } catch (final NumberFormatException e) { return; // malformed serial id — drop, never break eval } - final SpanEnrichmentAccumulator state = states.getOrCreate(traceKey); + final SpanEnrichmentAccumulator state = states.getOrCreate(root); state.addSerialId(serialId); if (doLog && targetingKey != null) { state.addSubject(targetingKey, serialId); } } else if (details.getVariant() == null) { // Runtime-default detection = MISSING VARIANT (never a reason enum). - states.getOrCreate(traceKey).addDefault(details.getFlagKey(), details.getValue()); + states.getOrCreate(root).addDefault(details.getFlagKey(), details.getValue()); } } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java index b9b65c839ab..988c04457c9 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java @@ -1,6 +1,5 @@ package datadog.trace.api.openfeature; -import datadog.trace.api.DDTraceId; import datadog.trace.api.interceptor.MutableSpan; import datadog.trace.api.interceptor.TraceInterceptor; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -50,8 +49,8 @@ * *

    State lives in the per-provider {@link SpanEnrichmentStates} store that is currently bound. * {@link #unbind(SpanEnrichmentStates)} clears only the store it unbinds, so one provider's close - * can never wipe another's in-flight state. The store is hard-bounded, so a trace that never - * reaches this interceptor cannot leak unboundedly. + * can never wipe another's in-flight state. The store is weak-keyed by the local-root span, so a + * trace that never reaches this interceptor is collected with its root and cannot leak unboundedly. * *

    All work is wrapped in try/catch — enrichment must NEVER break trace finish. */ @@ -133,13 +132,8 @@ public Collection onTraceComplete( if (!(localRoot instanceof AgentSpan)) { return trace; // partial flush, or no resolvable in-fragment root: keep state untouched } - final DDTraceId traceId = ((AgentSpan) localRoot).getTraceId(); - if (traceId == null) { - return trace; // no trace id (e.g. Noop span) — cannot key state; keep it - } - // Key by the full trace id (hex) to match the capture-side keying. - final String traceKey = traceId.toHexString(); - final SpanEnrichmentAccumulator state = states.remove(traceKey); + // Key by the local-root span object to match the capture-side keying. + final SpanEnrichmentAccumulator state = states.remove((AgentSpan) localRoot); if (state == null || !state.hasData()) { return trace; } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java index df120d8a6f2..50a7ab60db6 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java @@ -1,36 +1,27 @@ package datadog.trace.api.openfeature; -import java.util.Iterator; -import java.util.LinkedHashMap; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import java.util.Map; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.WeakHashMap; /** - * Bounded, instance-owned store of per-trace {@link SpanEnrichmentAccumulator} state, keyed by the - * local-root span's full trace id (the lower-case zero-padded hex string). + * Instance-owned store of per-trace {@link SpanEnrichmentAccumulator} state, keyed by the + * local-root span object. * - *

    The full hex key matters for 128-bit trace ids: keying by {@code DDTraceId.toLong()} (the - * low-order 64 bits only) would merge two distinct 128-bit traces that share their low bits into a - * single accumulator, cross-contaminating enrichment between unrelated traces. The full hex string - * is unique across all 128 bits and is cached on the id, so it is cheap to obtain. + *

    Weak keys. The map is a {@link WeakHashMap} keyed by the local-root {@link AgentSpan} + * instance. The accumulator is reachable only while its local-root span is, so a trace that never + * reaches the interceptor (dropped, Noop tracer, never-finishing root) is collected together with + * its root — it cannot leak unboundedly. No cap, FIFO eviction, or cleanup thread is required; + * {@code WeakHashMap} purges stale entries on access. * - *

    Each {@link SpanEnrichmentHook}/{@link SpanEnrichmentInterceptor} pair shares one store - * instance, so: - * - *

      - *
    • Unbounded leak: the store is hard-capped at {@link #MAX_TRACES}. A trace that never - * reaches the interceptor (dropped, Noop tracer, never-finishing root) can no longer leak - * unboundedly — once the cap is reached, the oldest in-flight entry is evicted (FIFO - * by insertion order) so the map size is strictly bounded regardless of trace completion. - *
    • Isolation: because the store is instance-owned rather than a shared static, one - * provider's cleanup clears only its own state and can never wipe another (still-active) - * provider's in-flight entries. - *
    + *

    Identity keying. {@code DDSpan} does not override {@code equals}/{@code hashCode}, so + * the map keys by object identity. Keying by the local-root span object (rather than the 128-bit + * trace-id hex string) is inherently unique per trace: two distinct traces have distinct root + * objects and can never share an accumulator. * - *

    Bounded eviction is intentionally lossy under pathological pressure: dropping the oldest - * accumulator degrades enrichment for that one (likely already-abandoned) trace rather than - * exhausting the heap. Enrichment correctness is best-effort by contract; heap safety is not. + *

    Each {@link SpanEnrichmentHook}/{@link SpanEnrichmentInterceptor} pair shares one store + * instance. Because the store is instance-owned rather than a shared static, one provider's cleanup + * clears only its own state and can never wipe another (still-active) provider's in-flight entries. * *

    Thread-safety: all access is guarded by the intrinsic lock on this instance. The hook writes * (eval thread) and the interceptor reads+removes (trace-write thread) concurrently, so every @@ -38,44 +29,24 @@ */ final class SpanEnrichmentStates { - private static final Logger log = LoggerFactory.getLogger(SpanEnrichmentStates.class); - - /** - * Hard cap on the number of concurrently-tracked traces. Sized well above any realistic count of - * simultaneously in-flight traces with active flag evaluations on a single JVM, so the live path - * never evicts; the cap exists purely to bound a leak of never-completing traces. - */ - static final int MAX_TRACES = 4096; - - // Insertion-ordered so the eldest entry is the natural eviction victim (FIFO). accessOrder=false - // (the default) — we evict by age of creation, not by recency of use, so a long-running but - // never-completing trace cannot pin the map by being repeatedly touched. - private final LinkedHashMap states = new LinkedHashMap<>(); - - // One-shot guard so a sustained leak logs once at WARN rather than on every eviction. - private boolean evictionWarned = false; + // Weak keys: the accumulator is GC'd with its local-root span, so a trace that never completes + // cannot leak. Identity-keyed by the local-root AgentSpan object. guarded by 'this'. + private final Map states = new WeakHashMap<>(); - /** - * Returns the accumulator for {@code traceKey}, creating (and inserting) it if absent. When - * insertion would exceed {@link #MAX_TRACES}, the eldest entry is evicted first so the store - * stays bounded. - */ - synchronized SpanEnrichmentAccumulator getOrCreate(final String traceKey) { - SpanEnrichmentAccumulator existing = states.get(traceKey); + /** Returns the accumulator for {@code root}, creating (and inserting) it if absent. */ + synchronized SpanEnrichmentAccumulator getOrCreate(final AgentSpan root) { + SpanEnrichmentAccumulator existing = states.get(root); if (existing != null) { return existing; } - if (states.size() >= MAX_TRACES) { - evictEldest(); - } final SpanEnrichmentAccumulator created = new SpanEnrichmentAccumulator(); - states.put(traceKey, created); + states.put(root, created); return created; } - /** Removes and returns the accumulator for {@code traceKey}, or {@code null} if absent. */ - synchronized SpanEnrichmentAccumulator remove(final String traceKey) { - return states.remove(traceKey); + /** Removes and returns the accumulator for {@code root}, or {@code null} if absent. */ + synchronized SpanEnrichmentAccumulator remove(final AgentSpan root) { + return states.remove(root); } /** Clears all tracked state (cleanup on provider close / unbind). */ @@ -93,23 +64,7 @@ synchronized boolean isEmpty() { // ---- test-only accessor ---- - synchronized SpanEnrichmentAccumulator peek(final String traceKey) { - return states.get(traceKey); - } - - private void evictEldest() { - final Iterator> it = states.entrySet().iterator(); - if (it.hasNext()) { - it.next(); - it.remove(); - } - if (!evictionWarned) { - evictionWarned = true; - log.warn( - "Span-enrichment state cap ({}) reached; evicting oldest in-flight trace state. " - + "This indicates traces with flag evaluations that never complete; enrichment for " - + "evicted traces is dropped to bound memory.", - MAX_TRACES); - } + synchronized SpanEnrichmentAccumulator peek(final AgentSpan root) { + return states.get(root); } } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java index b3d2a197d39..7912e4272c8 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java @@ -11,7 +11,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import datadog.trace.api.DDTraceId; import datadog.trace.api.interceptor.MutableSpan; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import dev.openfeature.sdk.FlagEvaluationDetails; @@ -61,9 +60,11 @@ void clearState() { // ---- helpers ---- - /** The store key the production code derives from a trace id (full hex). */ - private static String key(final long traceId) { - return DDTraceId.from(traceId).toHexString(); + /** A mock local-root span reporting itself as its own local root (identity key + final flush). */ + private static AgentSpan rootSpan() { + final AgentSpan root = mock(AgentSpan.class); + when(root.getLocalRootSpan()).thenReturn(root); + return root; } private static FlagEvaluationDetails details( @@ -98,10 +99,10 @@ private static HookContext ctx(final String flagKey, final String target "default"); } - /** Drives the capture branch directly (no static tracer) for a fixed trace id. */ + /** Drives the capture branch directly (no static tracer) for a fixed local-root span. */ private static void capture( final SpanEnrichmentHook hook, - final long traceId, + final AgentSpan root, final String flagKey, final String targetingKey, final String variant, @@ -109,22 +110,11 @@ private static void capture( final Integer serialId, final boolean doLog) { hook.capture( - key(traceId), + root, ctx(flagKey, targetingKey), details(flagKey, variant, value, metadata(serialId, doLog))); } - /** - * Configures a mock root span to report itself as its own local root with the given trace id. The - * returned span, passed in a singleton collection, models a FINAL flush (the root is present in - * the fragment), so the interceptor flushes + removes state. - */ - private static AgentSpan rootSpanCollection(final long traceId, final AgentSpan rootSpan) { - when(rootSpan.getLocalRootSpan()).thenReturn(rootSpan); - when(rootSpan.getTraceId()).thenReturn(DDTraceId.from(traceId)); - return rootSpan; - } - /** Binds a fresh interceptor to the given store, models a final flush, and returns it. */ private static SpanEnrichmentInterceptor boundInterceptor(final SpanEnrichmentStates states) { SpanEnrichmentInterceptor.INSTANCE.bind(states); @@ -170,13 +160,12 @@ void noActiveSpanDoesNotCrashOrAccumulate() { void finallyAfterResolvesRootViaResolverAndAccumulates() { // Drives finallyAfter end-to-end with an injected root resolver (no static mocks). final AgentSpan root = mock(AgentSpan.class); - when(root.getTraceId()).thenReturn(DDTraceId.from(0x77L)); final SpanEnrichmentHook hook = new SpanEnrichmentHook(() -> root, states); hook.finallyAfter( ctx("flag", "user-1"), details("flag", "on", "v", metadata(42, true)), Collections.emptyMap()); - final SpanEnrichmentAccumulator state = states.peek(key(0x77L)); + final SpanEnrichmentAccumulator state = states.peek(root); assertTrue(state != null && state.serialIdsView().contains(42)); assertEquals(1, state.subjectCount(), "doLog=true + targeting key => subject recorded"); } @@ -186,16 +175,14 @@ void finallyAfterResolvesRootViaResolverAndAccumulates() { @Test void finishedRootFlushesFlagsEncTag() { final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final long traceId = 0xABCDL; + final AgentSpan root = rootSpan(); // accumulate {100,108,128,130} with a duplicate 100 to prove dedupe - capture(hook, traceId, "f1", "user-1", "on", "v", 100, false); - capture(hook, traceId, "f2", "user-1", "on", "v", 108, false); - capture(hook, traceId, "f3", "user-1", "on", "v", 128, false); - capture(hook, traceId, "f4", "user-1", "on", "v", 130, false); - capture(hook, traceId, "f1", "user-1", "on", "v", 100, false); // dup + capture(hook, root, "f1", "user-1", "on", "v", 100, false); + capture(hook, root, "f2", "user-1", "on", "v", 108, false); + capture(hook, root, "f3", "user-1", "on", "v", 128, false); + capture(hook, root, "f4", "user-1", "on", "v", 130, false); + capture(hook, root, "f1", "user-1", "on", "v", 100, false); // dup - final AgentSpan root = mock(AgentSpan.class); - rootSpanCollection(traceId, root); boundInterceptor(states).onTraceComplete(Collections.singletonList(root)); verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZAgUAg=="); @@ -208,11 +195,11 @@ void finishedRootFlushesFlagsEncTag() { @Test void runtimeDefaultMissingVariantWritesJsonObject() { final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final long traceId = 0x1234L; + final AgentSpan root = rootSpan(); // no serial id + null variant => runtime default; object value must be JSON-stringified final Map objectValue = Collections.singletonMap("k", "val"); hook.capture( - key(traceId), + root, ctx("obj-flag", "user-1"), FlagEvaluationDetails.builder() .flagKey("obj-flag") @@ -221,8 +208,6 @@ void runtimeDefaultMissingVariantWritesJsonObject() { .flagMetadata(ImmutableMetadata.builder().build()) .build()); - final AgentSpan root = mock(AgentSpan.class); - rootSpanCollection(traceId, root); boundInterceptor(states).onTraceComplete(Collections.singletonList(root)); // ffe_runtime_defaults is a JSON object string, NOT [object Object]/toString. @@ -245,7 +230,7 @@ void runtimeDefaultMissingVariantWritesJsonObject() { @Test void runtimeDefaultStructureValueSerializesAsJsonNotToString() { final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final long traceId = 0x2468L; + final AgentSpan root = rootSpan(); // Single-key structure so the exact-string assertion does not depend on the OpenFeature SDK's // internal key ordering. (The multi-key / nesting behaviour is covered by the direct @@ -255,7 +240,7 @@ void runtimeDefaultStructureValueSerializesAsJsonNotToString() { final Value structureDefault = new Value(new ImmutableStructure(inner)); hook.capture( - key(traceId), + root, ctx("struct-flag", "user-1"), FlagEvaluationDetails.builder() .flagKey("struct-flag") @@ -264,8 +249,6 @@ void runtimeDefaultStructureValueSerializesAsJsonNotToString() { .flagMetadata(ImmutableMetadata.builder().build()) .build()); - final AgentSpan root = mock(AgentSpan.class); - rootSpanCollection(traceId, root); boundInterceptor(states).onTraceComplete(Collections.singletonList(root)); // {"struct-flag":"{\"enabled\":true}"} — note: NO "Value(innerObject=...)". @@ -316,12 +299,11 @@ void subjectCapsAndDoLogGating() { // doLog gating: addSubject only happens when doLog true (driven via the hook branch below). final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final long traceId = 0x5L; - capture(hook, traceId, "f", "user-A", "on", "v", 1, false); // doLog=false => no subject - assertEquals( - 0, states.peek(key(traceId)).subjectCount(), "doLog=false must not record a subject"); - capture(hook, traceId, "f", "user-A", "on", "v", 2, true); // doLog=true => subject recorded - assertEquals(1, states.peek(key(traceId)).subjectCount()); + final AgentSpan root = rootSpan(); + capture(hook, root, "f", "user-A", "on", "v", 1, false); // doLog=false => no subject + assertEquals(0, states.peek(root).subjectCount(), "doLog=false must not record a subject"); + capture(hook, root, "f", "user-A", "on", "v", 2, true); // doLog=true => subject recorded + assertEquals(1, states.peek(root).subjectCount()); // per-subject experiment cap: 20 max for (int i = 0; i < 25; i++) { @@ -448,7 +430,7 @@ void gateOnConstructsHookAndStateThenShutdownUnbinds() { // provider close unbinds + drains ITS OWN state. final SpanEnrichmentStates providerStates = provider.spanEnrichmentStates(); - providerStates.getOrCreate(key(1L)); + providerStates.getOrCreate(mock(AgentSpan.class)); assertFalse(providerStates.isEmpty()); provider.shutdown(); assertNull( diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java index 7e4d00b5875..d3a827617c2 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java @@ -1,5 +1,6 @@ package datadog.trace.api.openfeature; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -10,8 +11,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import datadog.trace.api.DD128bTraceId; -import datadog.trace.api.DDTraceId; import datadog.trace.api.interceptor.MutableSpan; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import dev.openfeature.sdk.EvaluationContext; @@ -20,6 +19,9 @@ import dev.openfeature.sdk.HookContext; import dev.openfeature.sdk.ImmutableContext; import dev.openfeature.sdk.ImmutableMetadata; +import java.lang.ref.WeakReference; +import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -36,12 +38,15 @@ * dd-trace-core) must NOT drain the accumulator; pre-flush flags must survive onto the root * at final completion. *
  • unbounded leak — traces that never reach the interceptor (dropped / Noop / never-finishing) - * must not leak accumulator entries unboundedly; the store is hard-bounded. + * must not leak accumulator entries unboundedly; the store is weak-keyed by the local-root + * span, so entries are collected once their root becomes unreachable (no cap / eviction). *
  • reconfiguration — after a provider closes, a new gate-on provider must still enrich (the * process-wide interceptor rebinds), and a closing provider must never clobber a newer * provider's in-flight state. - *
  • 128-bit trace ids — two distinct 128-bit traces that share their low-order 64 bits must not - * merge enrichment state. + *
  • distinct traces — two distinct traces (distinct local-root span objects) must not merge + * enrichment state. + *
  • keying symmetry — the root captured by the hook and the root resolved by the interceptor + * from the flushed fragment must be the SAME object, so identity keying round-trips. * */ class SpanEnrichmentLifecycleRegressionTest { @@ -54,10 +59,6 @@ void resetInterceptor() { // ---- helpers ---- - private static String key(final long traceId) { - return DDTraceId.from(traceId).toHexString(); - } - private static ImmutableMetadata serialMeta(final int serialId, final boolean doLog) { return ImmutableMetadata.builder() .addString(SpanEnrichmentHook.METADATA_SERIAL_ID, Integer.toString(serialId)) @@ -88,18 +89,13 @@ private static AgentSpan childOf(final AgentSpan root) { return child; } - /** A mock local-root span reporting itself as its own local root with the given trace id. */ - private static AgentSpan rootSpan(final DDTraceId traceId) { + /** A mock local-root span reporting itself as its own local root. */ + private static AgentSpan rootSpan() { final AgentSpan root = mock(AgentSpan.class); when(root.getLocalRootSpan()).thenReturn(root); - when(root.getTraceId()).thenReturn(traceId); return root; } - private static AgentSpan rootSpan(final long traceId) { - return rootSpan(DDTraceId.from(traceId)); - } - // ===================================================================================== // partial flush must not drop captured state or misattribute tags // ===================================================================================== @@ -125,12 +121,11 @@ void partialFlushExcludingRootPreservesPreFlushFlags() { final SpanEnrichmentInterceptor interceptor = SpanEnrichmentInterceptor.INSTANCE; interceptor.bind(states); - final long traceId = 0x10A6L; - final AgentSpan root = rootSpan(traceId); + final AgentSpan root = rootSpan(); // (1) pre-partial-flush evaluations - hook.capture(key(traceId), ctx("f1", "user-1"), serialDetails("f1", 100, false)); - hook.capture(key(traceId), ctx("f2", "user-1"), serialDetails("f2", 108, false)); + hook.capture(root, ctx("f1", "user-1"), serialDetails("f1", 100, false)); + hook.capture(root, ctx("f2", "user-1"), serialDetails("f2", 108, false)); // (2) PARTIAL flush: a fragment of children only — the open root is NOT in the collection. final AgentSpan child1 = childOf(root); @@ -142,15 +137,15 @@ void partialFlushExcludingRootPreservesPreFlushFlags() { verify(child1, never()).setTag(anyString(), anyString()); verify(child2, never()).setTag(anyString(), anyString()); verify(root, never()).setTag(anyString(), anyString()); - final SpanEnrichmentAccumulator surviving = states.peek(key(traceId)); + final SpanEnrichmentAccumulator surviving = states.peek(root); assertNotNull(surviving, "partial flush must NOT remove the accumulator"); assertTrue( surviving.serialIdsView().contains(100) && surviving.serialIdsView().contains(108), "pre-flush serial ids must survive a partial flush"); // (3) more evaluations after the partial flush - hook.capture(key(traceId), ctx("f3", "user-1"), serialDetails("f3", 128, false)); - hook.capture(key(traceId), ctx("f4", "user-1"), serialDetails("f4", 130, false)); + hook.capture(root, ctx("f3", "user-1"), serialDetails("f3", 128, false)); + hook.capture(root, ctx("f4", "user-1"), serialDetails("f4", 130, false)); // (4) FINAL flush: the root is present in the fragment (alongside a late child). final AgentSpan lateChild = childOf(root); @@ -173,16 +168,15 @@ void partialFlushNeverWritesTagsOnAChildSpan() { final SpanEnrichmentInterceptor interceptor = SpanEnrichmentInterceptor.INSTANCE; interceptor.bind(states); - final long traceId = 0xC0DEL; - final AgentSpan root = rootSpan(traceId); - hook.capture(key(traceId), ctx("f", "user-1"), serialDetails("f", 7, false)); + final AgentSpan root = rootSpan(); + hook.capture(root, ctx("f", "user-1"), serialDetails("f", 7, false)); final AgentSpan child = childOf(root); interceptor.onTraceComplete(Collections.singletonList(child)); verify(child, never()).setTag(anyString(), anyString()); verify(root, never()).setTag(anyString(), anyString()); - assertNotNull(states.peek(key(traceId)), "child-only fragment must keep state"); + assertNotNull(states.peek(root), "child-only fragment must keep state"); } // ===================================================================================== @@ -190,46 +184,54 @@ void partialFlushNeverWritesTagsOnAChildSpan() { // ===================================================================================== /** - * Simulates a sustained stream of traces that each evaluate a flag but never reach the - * interceptor (dropped traces / Noop tracer / never-finishing roots). The store must stay bounded - * by {@link SpanEnrichmentStates#MAX_TRACES} rather than growing without limit. + * There is no cap and no eviction: while their local-root spans are still reachable, every + * never-completing trace keeps its own accumulator. The weak-keyed store bounds memory by + * reachability (see {@link #unreachableRootStateIsWeaklyCollected()}), not by a fixed count, so + * inserting far more distinct roots than the old 4096 cap never drops a still-live entry. */ @Test - void neverCompletingTracesDoNotLeakUnbounded() { + void neverCompletingTracesAreNotCapped() { final SpanEnrichmentStates states = new SpanEnrichmentStates(); final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final int churn = SpanEnrichmentStates.MAX_TRACES * 3; + final int churn = 20_000; // well beyond the old 4096 cap + // Hold strong references so nothing is GC'd during the test — proves there is no size cap. + final List liveRoots = new ArrayList<>(churn); for (int i = 0; i < churn; i++) { - // Each distinct trace id accumulates once and is NEVER flushed (no interceptor call). - hook.capture(key(i), ctx("f", "user-" + i), serialDetails("f", i % 1000 + 1, false)); - assertTrue( - states.size() <= SpanEnrichmentStates.MAX_TRACES, - "state store must stay bounded for never-completing traces"); + final AgentSpan root = rootSpan(); + liveRoots.add(root); + hook.capture(root, ctx("f", "user-" + i), serialDetails("f", i % 1000 + 1, false)); } - assertEquals( - SpanEnrichmentStates.MAX_TRACES, states.size(), "store saturates at the cap, never beyond"); + assertEquals(churn, states.size(), "no cap/eviction: every live root keeps its accumulator"); } /** - * Bounding is FIFO by insertion: once the cap is reached, the oldest entries are evicted so the - * newest in-flight traces are retained. + * Weak-reference semantics: once a never-completing trace's local-root span becomes unreachable, + * its accumulator entry is collected — the store cannot leak unboundedly without any cap. GC is + * non-deterministic, so this polls with explicit {@code System.gc()} hints until the entry + * clears. */ @Test - void boundedStoreEvictsOldestFirst() { + void unreachableRootStateIsWeaklyCollected() { final SpanEnrichmentStates states = new SpanEnrichmentStates(); - // Fill exactly to the cap with ids [0, MAX). - for (long i = 0; i < SpanEnrichmentStates.MAX_TRACES; i++) { - states.getOrCreate(key(i)); - } - assertEquals(SpanEnrichmentStates.MAX_TRACES, states.size()); - // One more distinct id evicts the oldest (id 0) and keeps the rest + the new one. - final long overflow = SpanEnrichmentStates.MAX_TRACES; - states.getOrCreate(key(overflow)); - assertEquals(SpanEnrichmentStates.MAX_TRACES, states.size(), "size stays at the cap"); - assertNull(states.peek(key(0L)), "oldest entry evicted first (FIFO)"); - assertNotNull(states.peek(key(overflow)), "newest entry retained"); - assertNotNull(states.peek(key(1L)), "second-oldest still present"); + final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); + + AgentSpan root = rootSpan(); + hook.capture(root, ctx("f", "user-1"), serialDetails("f", 1, false)); + assertEquals(1, states.size(), "one in-flight accumulator before the root is dropped"); + + final WeakReference ref = new WeakReference<>(root); + root = null; // drop the only strong reference to the never-completing root + + await() + .atMost(Duration.ofSeconds(10)) + .pollInterval(Duration.ofMillis(50)) + .untilAsserted( + () -> { + System.gc(); + assertNull(ref.get(), "unreferenced root must be collectable"); + assertEquals(0, states.size(), "accumulator must be purged with its collected root"); + }); } // ===================================================================================== @@ -266,11 +268,8 @@ void newProviderAfterShutdownStillEnriches() { "the interceptor must rebind to the second provider's store"); // End-to-end: the second provider captures and the interceptor flushes onto the root. - final long traceId = 0xBEE5L; - second - .spanEnrichmentHook() - .capture(key(traceId), ctx("f", "user-1"), serialDetails("f", 5, false)); - final AgentSpan root = rootSpan(traceId); + final AgentSpan root = rootSpan(); + second.spanEnrichmentHook().capture(root, ctx("f", "user-1"), serialDetails("f", 5, false)); SpanEnrichmentInterceptor.INSTANCE.onTraceComplete(Collections.singletonList(root)); verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "BQ=="); // {5} -> 0x05 } @@ -289,11 +288,9 @@ void lateShutdownOfDisplacedProviderDoesNotClobberActiveProvider() { final SpanEnrichmentStates secondStates = second.spanEnrichmentStates(); // Second provider is now the active one and has live state. - final long liveTrace = 0xA11FEL; - second - .spanEnrichmentHook() - .capture(key(liveTrace), ctx("f", "user-1"), serialDetails("f", 9, false)); - assertNotNull(secondStates.peek(key(liveTrace)), "second provider has in-flight state"); + final AgentSpan root = rootSpan(); + second.spanEnrichmentHook().capture(root, ctx("f", "user-1"), serialDetails("f", 9, false)); + assertNotNull(secondStates.peek(root), "second provider has in-flight state"); assertTrue(SpanEnrichmentInterceptor.INSTANCE.activeStates() == secondStates); // The DISPLACED first provider shuts down late. The active (second) provider must be untouched. @@ -302,11 +299,10 @@ void lateShutdownOfDisplacedProviderDoesNotClobberActiveProvider() { SpanEnrichmentInterceptor.INSTANCE.activeStates() == secondStates, "late shutdown of a displaced provider must not unbind the active provider"); assertNotNull( - secondStates.peek(key(liveTrace)), + secondStates.peek(root), "late shutdown of a displaced provider must not clear the active provider's state"); // Sanity: the second provider still flushes correctly. - final AgentSpan root = rootSpan(liveTrace); SpanEnrichmentInterceptor.INSTANCE.onTraceComplete(Collections.singletonList(root)); verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "CQ=="); // {9} -> 0x09 } @@ -324,7 +320,7 @@ void eachProviderOwnsADistinctStateStore() { a.spanEnrichmentStates() != b.spanEnrichmentStates(), "providers must own distinct state stores"); - a.spanEnrichmentStates().getOrCreate(key(1L)); + a.spanEnrichmentStates().getOrCreate(rootSpan()); assertTrue(b.spanEnrichmentStates().isEmpty(), "stores are isolated"); } @@ -336,53 +332,80 @@ void eachProviderOwnsADistinctStateStore() { void hookAndInterceptorOfOneProviderShareTheSameStore() { final Provider provider = new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); - final long traceId = 0xBEEFL; - provider - .spanEnrichmentHook() - .capture(key(traceId), ctx("f", "user-1"), serialDetails("f", 9, false)); + final AgentSpan root = rootSpan(); + provider.spanEnrichmentHook().capture(root, ctx("f", "user-1"), serialDetails("f", 9, false)); assertNotNull( - SpanEnrichmentInterceptor.INSTANCE.activeStates().peek(key(traceId)), + SpanEnrichmentInterceptor.INSTANCE.activeStates().peek(root), "hook capture must be visible in the bound store (same instance)"); } // ===================================================================================== - // 128-bit trace ids that share their low-order 64 bits must not merge + // distinct traces (distinct local-root span objects) must not merge // ===================================================================================== /** - * Two distinct 128-bit trace ids whose low-order 64 bits are identical (so {@code toLong()} - * collides) must keep SEPARATE accumulators. Keying by {@code toLong()} would merge them; keying - * by the full hex string keeps them apart. + * Two distinct traces have two distinct local-root span objects, so identity keying keeps their + * accumulators SEPARATE regardless of any trace-id bits. (Under the old hex-string keying this + * guarded against 128-bit ids sharing their low-order 64 bits; identity keying makes it + * inherent.) */ @Test - void distinct128BitTraceIdsSharingLowBitsDoNotMerge() { + void distinctTracesDoNotMerge() { final SpanEnrichmentStates states = new SpanEnrichmentStates(); final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); final SpanEnrichmentInterceptor interceptor = SpanEnrichmentInterceptor.INSTANCE; interceptor.bind(states); - // Same low 64 bits (0xABCD), different high 64 bits — toLong() is identical for both. - final long lowBits = 0xABCDL; - final DDTraceId idA = DD128bTraceId.from(0x1111L, lowBits); - final DDTraceId idB = DD128bTraceId.from(0x2222L, lowBits); - assertEquals(idA.toLong(), idB.toLong(), "precondition: low 64 bits collide"); - assertTrue(!idA.toHexString().equals(idB.toHexString()), "precondition: full ids differ"); + final AgentSpan rootA = rootSpan(); + final AgentSpan rootB = rootSpan(); - // Capture distinct serial ids under each full trace id. - hook.capture(idA.toHexString(), ctx("fa", "user-A"), serialDetails("fa", 100, false)); - hook.capture(idB.toHexString(), ctx("fb", "user-B"), serialDetails("fb", 130, false)); + // Capture distinct serial ids under each distinct root. + hook.capture(rootA, ctx("fa", "user-A"), serialDetails("fa", 100, false)); + hook.capture(rootB, ctx("fb", "user-B"), serialDetails("fb", 130, false)); // They must NOT have merged into one accumulator. - assertEquals(2, states.size(), "distinct 128-bit traces must not share an accumulator"); + assertEquals(2, states.size(), "distinct root spans must not share an accumulator"); // Flush trace A: only {100} (not {100,130}). - final AgentSpan rootA = rootSpan(idA); interceptor.onTraceComplete(Collections.singletonList(rootA)); verify(rootA).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZA=="); // {100} -> 0x64 // Flush trace B: only {130}. - final AgentSpan rootB = rootSpan(idB); interceptor.onTraceComplete(Collections.singletonList(rootB)); verify(rootB).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ggE="); // {130} -> 0x82 0x01 } + + // ===================================================================================== + // keying symmetry: the hook's captured root and the interceptor's resolved root are identical + // ===================================================================================== + + /** + * The whole identity-keying design rests on the hook and the interceptor referencing the SAME + * local-root span object. The hook captures against the root resolved by its {@code + * RootSpanResolver}; the interceptor resolves the root from the flushed fragment via {@code + * findLocalRootInFragment}. This asserts both resolve to the same instance so the accumulator + * captured on the eval side is exactly the one removed and flushed on the write side. + */ + @Test + void hookCapturedRootAndInterceptorResolvedRootAreSameObject() { + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + final AgentSpan root = rootSpan(); + // The hook resolves its root via the injected resolver (the production DEFAULT_RESOLVER calls + // activeSpan().getLocalRootSpan(); here we inject the same object the fragment will report). + final SpanEnrichmentHook hook = new SpanEnrichmentHook(() -> root, states); + final SpanEnrichmentInterceptor interceptor = SpanEnrichmentInterceptor.INSTANCE; + interceptor.bind(states); + + hook.finallyAfter( + ctx("f", "user-1"), serialDetails("f", 5, false), Collections.emptyMap()); + final SpanEnrichmentAccumulator captured = states.peek(root); + assertNotNull(captured, "hook captures against the resolved local root"); + + // The interceptor resolves the root from the fragment; it must be the same object, so the + // remove hits the captured accumulator and flushes it onto that exact root. + final AgentSpan child = childOf(root); + interceptor.onTraceComplete(Arrays.asList(child, root)); + verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "BQ=="); // {5} -> 0x05 + assertTrue(states.isEmpty(), "the interceptor removed the same accumulator the hook created"); + } } From 99422b5c34bba22b17a6a82127e286e89b615320 Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 7 Jul 2026 16:47:31 +0300 Subject: [PATCH 10/18] Enhancements after techdebt skill --- .../SpanEnrichmentAccumulator.java | 7 ++-- .../trace/api/openfeature/ULeb128Encoder.java | 32 ------------------ .../openfeature/SpanEnrichmentHookTest.java | 33 +++++++++++++++++-- 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java index 76265a89c50..594650ac08c 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java @@ -261,10 +261,9 @@ private static String toJsonValue(final Object value) { } @SuppressWarnings("unchecked") - private static void appendJsonValue(final StringBuilder sb, final Object rawValue) { - // Unwrap any OpenFeature Value to its native representation first so nested structures/lists - // serialize as JSON rather than via Value.toString(). - final Object value = rawValue instanceof Value ? unwrapValue((Value) rawValue) : rawValue; + private static void appendJsonValue(final StringBuilder sb, final Object value) { + // Callers pass values already unwrapped to native form by stringifyDefault/unwrapValue, so no + // OpenFeature Value ever reaches here. if (value == null) { sb.append("null"); } else if (value instanceof Map) { diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java index c256a829150..14e441a28b4 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java @@ -59,38 +59,6 @@ static String encodeDeltaVarint(final Set serialIds) { return Base64.getEncoder().encodeToString(out); } - /** - * Decodes a delta-varint base64 string back into the original ascending serial ids. Used by the - * codec round-trip oracle (the decode side mirrors the L2 system-tests decoder). - * - * @param encoded a base64 string produced by {@link #encodeDeltaVarint(Set)} - * @return the decoded serial ids in ascending order (empty for the empty string) - */ - static SortedSet decodeDeltaVarint(final String encoded) { - final SortedSet result = new TreeSet<>(); - if (encoded == null || encoded.isEmpty()) { - return result; - } - final byte[] bytes = Base64.getDecoder().decode(encoded); - int previous = 0; - int index = 0; - while (index < bytes.length) { - long value = 0; - int shift = 0; - while (true) { - final byte b = bytes[index++]; - value |= ((long) (b & 0x7F)) << shift; - if ((b & 0x80) == 0) { - break; - } - shift += 7; - } - previous += (int) value; // delta from previous - result.add(previous); - } - return result; - } - /** * Lower-case hex SHA-256 of the given string. Used to hash subject targeting keys before they are * emitted (privacy: subject keys are never emitted in clear text). diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java index 7912e4272c8..183b867f1e9 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java @@ -22,6 +22,7 @@ import dev.openfeature.sdk.ImmutableStructure; import dev.openfeature.sdk.Value; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -133,7 +134,7 @@ void codecGoldenVectorAndRoundTrip() { final String encoded = ULeb128Encoder.encodeDeltaVarint(ids); assertEquals("ZAgUAg==", encoded, "golden vector must match the frozen Node contract"); // round-trip: decode back to the same ascending ids - assertEquals(ids, ULeb128Encoder.decodeDeltaVarint(encoded)); + assertEquals(ids, decodeDeltaVarint(encoded)); // empty set -> empty string (tag omitted) assertEquals("", ULeb128Encoder.encodeDeltaVarint(Collections.emptySet())); // dedupe is structural: a duplicate id does not change the encoding @@ -142,6 +143,34 @@ void codecGoldenVectorAndRoundTrip() { assertEquals(encoded, ULeb128Encoder.encodeDeltaVarint(withDup)); } + // Test-only decode oracle for the ULEB128 delta-varint codec. The encoder ships in the published + // dd-openfeature jar; the decode side exists only to assert round-trips here, so it lives in the + // test rather than in production code. + private static SortedSet decodeDeltaVarint(final String encoded) { + final SortedSet result = new TreeSet<>(); + if (encoded == null || encoded.isEmpty()) { + return result; + } + final byte[] bytes = Base64.getDecoder().decode(encoded); + int previous = 0; + int index = 0; + while (index < bytes.length) { + long value = 0; + int shift = 0; + while (true) { + final byte b = bytes[index++]; + value |= ((long) (b & 0x7F)) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + previous += (int) value; // delta from previous + result.add(previous); + } + return result; + } + // ---- 2. no-span (no active root) ---- @Test @@ -311,7 +340,7 @@ void subjectCapsAndDoLogGating() { } final Map tags = acc.toSpanTags(); final SortedSet decoded = - ULeb128Encoder.decodeDeltaVarint( + decodeDeltaVarint( // ffe_subjects_enc is {"":""}; extract the single base64 value tags.get(SpanEnrichmentAccumulator.TAG_SUBJECTS_ENC) .replaceAll("^\\{\"[a-f0-9]+\":\"", "") From 4be86c66f867ff72944d744a8942acb490fb8cb4 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 8 Jul 2026 11:37:15 +0300 Subject: [PATCH 11/18] Refactor: Move span-enrichment write tier out of the published dd-openfeature API --- .../featureflag/FeatureFlaggingSystem.java | 11 + .../feature-flagging-api/build.gradle.kts | 11 +- .../trace/api/openfeature/Provider.java | 82 +-- .../api/openfeature/SpanEnrichmentHook.java | 187 ++++--- .../openfeature/SpanEnrichmentHookTest.java | 480 ++++-------------- ...SpanEnrichmentLifecycleRegressionTest.java | 411 --------------- .../featureflag/FeatureFlaggingGateway.java | 16 + .../api/featureflag/SpanEnrichmentEvent.java | 82 +++ .../feature-flagging-lib/build.gradle.kts | 5 + .../SpanEnrichmentAccumulator.java | 111 +--- .../SpanEnrichmentInterceptor.java | 99 +--- .../featureflag}/SpanEnrichmentStates.java | 15 +- .../featureflag/SpanEnrichmentWriter.java | 121 +++++ .../datadog/featureflag}/ULeb128Encoder.java | 2 +- .../SpanEnrichmentAccumulatorTest.java | 159 ++++++ .../SpanEnrichmentInterceptorTest.java | 162 ++++++ .../featureflag/SpanEnrichmentWriterTest.java | 114 +++++ 17 files changed, 957 insertions(+), 1111 deletions(-) delete mode 100644 products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java create mode 100644 products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/SpanEnrichmentEvent.java rename products/feature-flagging/{feature-flagging-api/src/main/java/datadog/trace/api/openfeature => feature-flagging-lib/src/main/java/com/datadog/featureflag}/SpanEnrichmentAccumulator.java (69%) rename products/feature-flagging/{feature-flagging-api/src/main/java/datadog/trace/api/openfeature => feature-flagging-lib/src/main/java/com/datadog/featureflag}/SpanEnrichmentInterceptor.java (53%) rename products/feature-flagging/{feature-flagging-api/src/main/java/datadog/trace/api/openfeature => feature-flagging-lib/src/main/java/com/datadog/featureflag}/SpanEnrichmentStates.java (81%) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java rename products/feature-flagging/{feature-flagging-api/src/main/java/datadog/trace/api/openfeature => feature-flagging-lib/src/main/java/com/datadog/featureflag}/ULeb128Encoder.java (98%) create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentAccumulatorTest.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentInterceptorTest.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentWriterTest.java diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 02689767bad..b6d77065780 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -11,6 +11,7 @@ public class FeatureFlaggingSystem { private static volatile RemoteConfigService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; + private static volatile SpanEnrichmentWriter SPAN_ENRICHMENT_WRITER; private FeatureFlaggingSystem() {} @@ -27,10 +28,20 @@ public static void start(final SharedCommunicationObjects sco) { EXPOSURE_WRITER = new ExposureWriterImpl(sco, config); EXPOSURE_WRITER.init(); + // APM span enrichment: agent-side listener for flag-evaluation seam events. Cheap to register + // (it only accumulates once the provider's gate-on capture hook actually dispatches events, and + // registers its trace interceptor lazily on the first such event). + SPAN_ENRICHMENT_WRITER = new SpanEnrichmentWriter(); + SPAN_ENRICHMENT_WRITER.init(); + LOGGER.debug("Feature Flagging system started"); } public static void stop() { + if (SPAN_ENRICHMENT_WRITER != null) { + SPAN_ENRICHMENT_WRITER.close(); + SPAN_ENRICHMENT_WRITER = null; + } if (EXPOSURE_WRITER != null) { EXPOSURE_WRITER.close(); EXPOSURE_WRITER = null; diff --git a/products/feature-flagging/feature-flagging-api/build.gradle.kts b/products/feature-flagging/feature-flagging-api/build.gradle.kts index 20539543fcc..2d6308592cb 100644 --- a/products/feature-flagging/feature-flagging-api/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-api/build.gradle.kts @@ -45,16 +45,15 @@ dependencies { compileOnly(project(":products:feature-flagging:feature-flagging-bootstrap")) compileOnly(project(":utils:config-utils")) - // Span enrichment: TraceInterceptor / GlobalTracer / AgentTracer / AgentSpan for the - // write tier + active-root-span lookup. compileOnly because the agent runtime - // (feature-flagging-agent depends on :internal-api) provides these classes; the published - // dd-openfeature jar must not bundle the tracer. - compileOnly(project(":internal-api")) + // Public config constants (FeatureFlaggingConfig). dd-trace-api is the customer-facing public API, + // so this is a valid product-API dependency (previously reached transitively via :internal-api). + // compileOnly: the constant is inlined, and the agent runtime provides the class. + compileOnly(project(":dd-trace-api")) compileOnly("io.opentelemetry:opentelemetry-api:1.47.0") testImplementation(project(":products:feature-flagging:feature-flagging-bootstrap")) - testImplementation(project(":internal-api")) testImplementation(project(":utils:config-utils")) + testImplementation(project(":dd-trace-api")) testImplementation("io.opentelemetry:opentelemetry-api:1.47.0") testImplementation(libs.bundles.junit5) testImplementation(libs.bundles.mockito) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index 5cc32e7553b..21df800d703 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -2,7 +2,6 @@ import static java.util.concurrent.TimeUnit.SECONDS; -import datadog.trace.api.GlobalTracer; import datadog.trace.api.config.FeatureFlaggingConfig; import datadog.trace.bootstrap.config.provider.ConfigProvider; import de.thetaphi.forbiddenapis.SuppressForbidden; @@ -52,7 +51,6 @@ public class Provider extends EventProvider implements Metadata { private final FlagEvalHook flagEvalHook; // Span enrichment: null unless the gate is on, so the feature has no idle overhead when off. private final SpanEnrichmentHook spanEnrichmentHook; - private final SpanEnrichmentStates spanEnrichmentStates; // Precomputed hook list returned by getProviderHooks() on every evaluation. Immutable and built // once so gate-off evaluation allocates nothing on this hot path. private final List providerHooks; @@ -69,17 +67,6 @@ public Provider(final Options options) { this(options, evaluator, null); } - /** - * Registers a {@link SpanEnrichmentInterceptor} with the running tracer, returning {@code true} - * when it was added and {@code false} when the tracer rejected it (e.g. the interceptor is - * already registered, or the global tracer is the no-op placeholder). Injectable so tests can - * drive registration deterministically without mutating the global tracer (mirrors the {@code - * spanEnrichmentEnabledOverride} seam). - */ - interface TraceInterceptorRegistrar { - boolean register(SpanEnrichmentInterceptor interceptor); - } - /** * @param spanEnrichmentEnabledOverride when non-null, forces the span-enrichment gate (test * seam); when null, the gate is read from {@link #SPAN_ENRICHMENT_ENABLED_KEY}. @@ -88,22 +75,6 @@ interface TraceInterceptorRegistrar { final Options options, final Evaluator evaluator, final Boolean spanEnrichmentEnabledOverride) { - this( - options, - evaluator, - spanEnrichmentEnabledOverride, - interceptor -> GlobalTracer.get().addTraceInterceptor(interceptor)); - } - - /** - * @param registrar registers the span-enrichment interceptor with the tracer; injectable for - * tests (see {@link TraceInterceptorRegistrar}). - */ - Provider( - final Options options, - final Evaluator evaluator, - final Boolean spanEnrichmentEnabledOverride, - final TraceInterceptorRegistrar registrar) { this.options = options; this.evaluator = evaluator; FlagEvalMetrics metrics = null; @@ -118,34 +89,15 @@ interface TraceInterceptorRegistrar { this.flagEvalMetrics = metrics; this.flagEvalHook = hook; - // Span enrichment is wired ONLY when the gate is on. When off, no hook/state is constructed and - // there is no idle per-evaluation or per-span overhead. + // Span enrichment is wired ONLY when the gate is on. When off, no capture hook is constructed + // and there is no idle per-evaluation overhead. The hook merely dispatches evaluation metadata + // onto FeatureFlaggingGateway; the agent-side write tier (feature-flagging-lib) resolves the + // span and accumulates, so this application-side provider holds no tracer dependency. final boolean spanEnrichmentEnabled = spanEnrichmentEnabledOverride != null ? spanEnrichmentEnabledOverride : isSpanEnrichmentEnabled(); - SpanEnrichmentHook seHook = null; - SpanEnrichmentStates seStates = null; - if (spanEnrichmentEnabled) { - try { - // Per-provider state store, shared with this provider's capture hook. The single, - // process-wide interceptor is registered once (reconfiguration-safe) and rebound to this - // provider's store. A later gate-on provider rebinds it to its own store; this provider's - // shutdown only unbinds if it is still the active provider, so reconfiguration never - // permanently disables enrichment and providers never clobber each other's live state. - seStates = new SpanEnrichmentStates(); - SpanEnrichmentInterceptor.ensureRegistered(registrar); - SpanEnrichmentInterceptor.INSTANCE.bind(seStates); - seHook = new SpanEnrichmentHook(seStates); - } catch (LinkageError | Exception e) { - // Tracer classes absent (e.g. API-only classpath): degrade to no span enrichment. - log.warn("Span enrichment unavailable — tracer classes not on classpath", e); - seHook = null; - seStates = null; - } - } - this.spanEnrichmentHook = seHook; - this.spanEnrichmentStates = seStates; + this.spanEnrichmentHook = spanEnrichmentEnabled ? new SpanEnrichmentHook() : null; // Precompute the immutable hook list once so getProviderHooks() (called on every evaluation) // allocates nothing, including when the gate is off. @@ -153,15 +105,14 @@ interface TraceInterceptorRegistrar { if (flagEvalHook != null) { hooks.add(flagEvalHook); } - if (seHook != null) { - hooks.add(seHook); + if (spanEnrichmentHook != null) { + hooks.add(spanEnrichmentHook); } this.providerHooks = hooks.isEmpty() ? Collections.emptyList() : Collections.unmodifiableList(hooks); // Announce the span-enrichment state at startup (matches the reference implementation). - // Reflects effective wiring: "enabled" only when the hook was actually constructed (gate on AND - // tracer classes present), otherwise "disabled". + // "enabled" only when the gate is on (the capture hook was constructed), otherwise "disabled". if (spanEnrichmentHook != null) { log.info("{} span enrichment enabled", METADATA); } else { @@ -294,28 +245,19 @@ public void shutdown() { if (flagEvalMetrics != null) { flagEvalMetrics.shutdown(); } - // Provider-close cleanup for span enrichment: the tracer has no interceptor-removal API, so we - // unbind this provider's store from the process-wide interceptor (which clears the store and - // makes the interceptor inert until a new provider rebinds it). The unbind is a no-op if a - // newer provider has already rebound the interceptor, so we never wipe another provider's - // in-flight state. - if (spanEnrichmentStates != null) { - SpanEnrichmentInterceptor.INSTANCE.unbind(spanEnrichmentStates); - } + // Span enrichment needs no provider-close cleanup here: the capture hook holds no tracer state. + // The agent-side write tier owns the interceptor and per-trace state and is torn down with the + // feature-flagging subsystem, not per provider. if (evaluator != null) { evaluator.shutdown(); } } - // Visible for tests: expose whether span enrichment is wired (gate-on) without leaking the impls. + // Visible for tests: expose whether span enrichment is wired (gate-on) without leaking the impl. SpanEnrichmentHook spanEnrichmentHook() { return spanEnrichmentHook; } - SpanEnrichmentStates spanEnrichmentStates() { - return spanEnrichmentStates; - } - @Override public Metadata getMetadata() { return this; diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java index 6fe84b6d1e6..ef4ce123bc9 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java @@ -1,33 +1,40 @@ package datadog.trace.api.openfeature; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.SpanEnrichmentEvent; import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.FlagEvaluationDetails; import dev.openfeature.sdk.Hook; import dev.openfeature.sdk.HookContext; import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.Structure; +import dev.openfeature.sdk.Value; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; /** - * OpenFeature {@code finally} hook that captures feature-flag evaluation metadata into - * per-local-root span state for APM span enrichment. This is the CAPTURE half of the - * capture-vs-write split — the WRITE half is {@link SpanEnrichmentInterceptor}, which flushes the - * accumulated tags onto the local root span when the trace completes. + * OpenFeature {@code finally} hook that captures feature-flag evaluation metadata for APM span + * enrichment. This is the CAPTURE half of the capture-vs-write split, and it runs in the + * application classloader (returned from {@link Provider#getProviderHooks()} only when the gate is + * on). * - *

    Mirrors {@link FlagEvalHook}: registered via {@link Provider#getProviderHooks()} (only when - * the span-enrichment gate is on) and reading {@code details.getFlagMetadata()}. It resolves the - * active local-root span via {@link AgentTracer#activeSpan()} and keys the {@link - * SpanEnrichmentStates} store (shared with the {@link SpanEnrichmentInterceptor}) by that root span - * object, so the interceptor running later on the write thread can recover the same state from the - * completed span collection (the local root is a single stable object for the trace). + *

    It holds no tracer dependency: rather than resolving the active span itself, it emits a + * {@link SpanEnrichmentEvent} onto {@link FeatureFlaggingGateway}. The agent-side write tier + * ({@code SpanEnrichmentWriter} in {@code feature-flagging-lib}) receives the event, resolves the + * active local-root span, and accumulates the per-trace state that is flushed onto the root when + * the trace completes. This keeps the published {@code dd-openfeature} product API free of {@code + * internal-api}. * *

    Capture branch (frozen Node reference): * *

      - *
    • serial id present → addSerialId, plus addSubject when {@code __dd_do_log} AND a targeting - * key - *
    • else variant missing (runtime default) → addDefault(flagKey, value) + *
    • serial id present → dispatch a serial-id event (the write side records the serial id, plus + * the subject when {@code __dd_do_log} AND a targeting key) + *
    • else variant missing (runtime default) → dispatch a runtime-default event with the value + * unwrapped to a native Java type *
    * *

    All work is wrapped in try/catch — enrichment must NEVER break flag evaluation. @@ -37,36 +44,6 @@ class SpanEnrichmentHook implements Hook { static final String METADATA_SERIAL_ID = "__dd_split_serial_id"; static final String METADATA_DO_LOG = "__dd_do_log"; - /** - * Resolves the local-root span for the active trace. Injectable so tests need no static mocks. - */ - interface RootSpanResolver { - AgentSpan activeLocalRoot(); - } - - private static final RootSpanResolver DEFAULT_RESOLVER = - () -> { - final AgentSpan active = AgentTracer.activeSpan(); - if (active == null) { - return null; - } - final AgentSpan localRoot = active.getLocalRootSpan(); - return localRoot != null ? localRoot : active; - }; - - private final RootSpanResolver rootSpanResolver; - // State store shared with the interceptor. The hook writes here; the interceptor reads + removes. - private final SpanEnrichmentStates states; - - SpanEnrichmentHook(final SpanEnrichmentStates states) { - this(DEFAULT_RESOLVER, states); - } - - SpanEnrichmentHook(final RootSpanResolver rootSpanResolver, final SpanEnrichmentStates states) { - this.rootSpanResolver = rootSpanResolver; - this.states = states; - } - @Override public void finallyAfter( final HookContext ctx, @@ -76,49 +53,31 @@ public void finallyAfter( return; } try { - final AgentSpan root = rootSpanResolver.activeLocalRoot(); - if (root == null) { - return; // no active span → nothing to enrich + final ImmutableMetadata metadata = details.getFlagMetadata(); + final String serialIdStr = metadata != null ? metadata.getString(METADATA_SERIAL_ID) : null; + if (serialIdStr != null) { + final int serialId; + try { + serialId = Integer.parseInt(serialIdStr); + } catch (final NumberFormatException e) { + return; // malformed serial id — drop, never break eval + } + final String doLogStr = metadata.getString(METADATA_DO_LOG); + final boolean doLog = "true".equalsIgnoreCase(doLogStr); + FeatureFlaggingGateway.dispatch( + SpanEnrichmentEvent.serialId(serialId, doLog, targetingKey(ctx))); + } else if (details.getVariant() == null) { + // Runtime-default detection = MISSING VARIANT (never a reason enum). Unwrap any OpenFeature + // Value to a native Java type here so the seam carries only JDK types. + FeatureFlaggingGateway.dispatch( + SpanEnrichmentEvent.runtimeDefault( + details.getFlagKey(), unwrapDefaultValue(details.getValue()))); } - capture(root, ctx, details); } catch (final Throwable t) { // Never let span enrichment break flag evaluation. } } - /** - * Applies the frozen Node capture branch against the state keyed by the local-root {@code root} - * span. Package private so it can be driven deterministically in tests without stubbing the - * static tracer. - */ - void capture( - final AgentSpan root, - final HookContext ctx, - final FlagEvaluationDetails details) { - final ImmutableMetadata metadata = details.getFlagMetadata(); - final String serialIdStr = metadata != null ? metadata.getString(METADATA_SERIAL_ID) : null; - final String doLogStr = metadata != null ? metadata.getString(METADATA_DO_LOG) : null; - final boolean doLog = "true".equalsIgnoreCase(doLogStr); - final String targetingKey = targetingKey(ctx); - - if (serialIdStr != null) { - final int serialId; - try { - serialId = Integer.parseInt(serialIdStr); - } catch (final NumberFormatException e) { - return; // malformed serial id — drop, never break eval - } - final SpanEnrichmentAccumulator state = states.getOrCreate(root); - state.addSerialId(serialId); - if (doLog && targetingKey != null) { - state.addSubject(targetingKey, serialId); - } - } else if (details.getVariant() == null) { - // Runtime-default detection = MISSING VARIANT (never a reason enum). - states.getOrCreate(root).addDefault(details.getFlagKey(), details.getValue()); - } - } - private static String targetingKey(final HookContext ctx) { if (ctx == null) { return null; @@ -130,4 +89,68 @@ private static String targetingKey(final HookContext ctx) { final String key = evaluationContext.getTargetingKey(); return (key == null || key.isEmpty()) ? null : key; } + + /** + * Unwraps an OpenFeature {@link Value} to a native Java representation so the runtime default can + * cross the (JDK-types-only) seam and be JSON-serialized on the write side exactly like Node's + * {@code JSON.stringify}. Non-{@code Value} inputs (already native) are returned as-is. + */ + static Object unwrapDefaultValue(final Object value) { + return value instanceof Value ? unwrapValue((Value) value) : value; + } + + /** + * Recursively unwraps an OpenFeature {@link Value} into its native Java representation: + * structures become {@code Map}, lists become {@code List}, and scalars + * become their boxed value (or {@code null}). Nested {@link Value}s are unwrapped at every level + * so a structure containing further structures/lists serializes correctly. + */ + private static Object unwrapValue(final Value value) { + if (value == null || value.isNull()) { + return null; + } + if (value.isStructure()) { + final Structure structure = value.asStructure(); + final Map map = new LinkedHashMap<>(); + if (structure != null) { + for (final String key : structure.keySet()) { + map.put(key, unwrapValue(structure.getValue(key))); + } + } + return map; + } + if (value.isList()) { + final List list = value.asList(); + final List out = new ArrayList<>(list == null ? 0 : list.size()); + if (list != null) { + for (final Value element : list) { + out.add(unwrapValue(element)); + } + } + return out; + } + if (value.isBoolean()) { + return value.asBoolean(); + } + if (value.isString()) { + return value.asString(); + } + if (value.isNumber()) { + // Preserve integral vs fractional so the rendered JSON number matches Node. + final Double d = value.asDouble(); + if (d != null && d == Math.rint(d) && !Double.isInfinite(d)) { + final Integer i = value.asInteger(); + if (i != null) { + return i; + } + } + return d; + } + final Instant instant = value.asInstant(); + if (instant != null) { + return instant.toString(); + } + // Unknown shape: fall back to the wrapped object's own representation. + return value.asObject(); + } } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java index 183b867f1e9..3ed05b94abd 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java @@ -2,17 +2,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import datadog.trace.api.interceptor.MutableSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; + +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.SpanEnrichmentEvent; import dev.openfeature.sdk.FlagEvaluationDetails; import dev.openfeature.sdk.FlagValueType; import dev.openfeature.sdk.Hook; @@ -21,53 +16,42 @@ import dev.openfeature.sdk.ImmutableMetadata; import dev.openfeature.sdk.ImmutableStructure; import dev.openfeature.sdk.Value; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Base64; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; /** - * Unit suite for APM feature-flag span enrichment. + * Capture-side unit suite for APM feature-flag span enrichment. * - *

    Covers the seven required validation cases plus the explicit max-200 case and the codec - * golden-vector round-trip. The contract (encoding, limits, tag shapes) is FROZEN against the Node - * reference ({@code dd-trace-js#8343}). + *

    The published {@code dd-openfeature} provider has no tracer dependency: the hook dispatches a + * {@link SpanEnrichmentEvent} onto {@link FeatureFlaggingGateway} (native JDK types only) and the + * agent-side write tier does the accumulation. These tests therefore assert the dispatched events + * (not span tags — those are covered in {@code feature-flagging-lib}) plus the {@link Provider} + * gating. */ class SpanEnrichmentHookTest { - // Instance-owned state store shared by the hook and interceptor under test, mirroring how a - // single - // Provider wires them. - private SpanEnrichmentStates states; + private final List captured = new ArrayList<>(); + private final FeatureFlaggingGateway.SpanEnrichmentListener listener = captured::add; @BeforeEach - void freshState() { - states = new SpanEnrichmentStates(); + void register() { + FeatureFlaggingGateway.addSpanEnrichmentListener(listener); } @AfterEach - void clearState() { - states.clear(); - // The interceptor is a process-wide singleton; leave it inert for the next test. - SpanEnrichmentInterceptor.INSTANCE.unbind(SpanEnrichmentInterceptor.INSTANCE.activeStates()); + void deregister() { + FeatureFlaggingGateway.removeSpanEnrichmentListener(listener); } // ---- helpers ---- - /** A mock local-root span reporting itself as its own local root (identity key + final flush). */ - private static AgentSpan rootSpan() { - final AgentSpan root = mock(AgentSpan.class); - when(root.getLocalRootSpan()).thenReturn(root); - return root; - } - private static FlagEvaluationDetails details( final String flagKey, final String variant, @@ -100,355 +84,141 @@ private static HookContext ctx(final String flagKey, final String target "default"); } - /** Drives the capture branch directly (no static tracer) for a fixed local-root span. */ - private static void capture( - final SpanEnrichmentHook hook, - final AgentSpan root, - final String flagKey, - final String targetingKey, - final String variant, - final Object value, - final Integer serialId, - final boolean doLog) { - hook.capture( - root, - ctx(flagKey, targetingKey), - details(flagKey, variant, value, metadata(serialId, doLog))); - } - - /** Binds a fresh interceptor to the given store, models a final flush, and returns it. */ - private static SpanEnrichmentInterceptor boundInterceptor(final SpanEnrichmentStates states) { - SpanEnrichmentInterceptor.INSTANCE.bind(states); - return SpanEnrichmentInterceptor.INSTANCE; - } - - // ---- 1. codec golden-vector round-trip ---- + // ---- serial-id branch ---- @Test - void codecGoldenVectorAndRoundTrip() { - final SortedSet ids = new TreeSet<>(); - ids.add(100); - ids.add(108); - ids.add(128); - ids.add(130); - final String encoded = ULeb128Encoder.encodeDeltaVarint(ids); - assertEquals("ZAgUAg==", encoded, "golden vector must match the frozen Node contract"); - // round-trip: decode back to the same ascending ids - assertEquals(ids, decodeDeltaVarint(encoded)); - // empty set -> empty string (tag omitted) - assertEquals("", ULeb128Encoder.encodeDeltaVarint(Collections.emptySet())); - // dedupe is structural: a duplicate id does not change the encoding - final SortedSet withDup = new TreeSet<>(ids); - withDup.add(100); - assertEquals(encoded, ULeb128Encoder.encodeDeltaVarint(withDup)); - } + void serialIdWithDoLogDispatchesSerialAndSubject() { + new SpanEnrichmentHook() + .finallyAfter( + ctx("flag", "user-1"), + details("flag", "on", "v", metadata(42, true)), + Collections.emptyMap()); - // Test-only decode oracle for the ULEB128 delta-varint codec. The encoder ships in the published - // dd-openfeature jar; the decode side exists only to assert round-trips here, so it lives in the - // test rather than in production code. - private static SortedSet decodeDeltaVarint(final String encoded) { - final SortedSet result = new TreeSet<>(); - if (encoded == null || encoded.isEmpty()) { - return result; - } - final byte[] bytes = Base64.getDecoder().decode(encoded); - int previous = 0; - int index = 0; - while (index < bytes.length) { - long value = 0; - int shift = 0; - while (true) { - final byte b = bytes[index++]; - value |= ((long) (b & 0x7F)) << shift; - if ((b & 0x80) == 0) { - break; - } - shift += 7; - } - previous += (int) value; // delta from previous - result.add(previous); - } - return result; + assertEquals(1, captured.size()); + final SpanEnrichmentEvent event = captured.get(0); + assertTrue(event.hasSerialId()); + assertEquals(42, event.serialId()); + assertTrue(event.doLog()); + assertEquals("user-1", event.targetingKey()); } - // ---- 2. no-span (no active root) ---- - @Test - void noActiveSpanDoesNotCrashOrAccumulate() { - // Injected resolver returns null => no active local root (the no-span case). - final SpanEnrichmentHook hook = new SpanEnrichmentHook(() -> null, states); - // Should be a no-op, never throw. - hook.finallyAfter( - ctx("flag", "user-1"), - details("flag", "on", "v", metadata(100, true)), - Collections.emptyMap()); - assertTrue(states.isEmpty(), "no active span => no accumulator state"); - } + void serialIdWithoutDoLogStillDispatchesSerialButNotDoLog() { + new SpanEnrichmentHook() + .finallyAfter( + ctx("flag", "user-1"), + details("flag", "on", "v", metadata(7, false)), + Collections.emptyMap()); - @Test - void finallyAfterResolvesRootViaResolverAndAccumulates() { - // Drives finallyAfter end-to-end with an injected root resolver (no static mocks). - final AgentSpan root = mock(AgentSpan.class); - final SpanEnrichmentHook hook = new SpanEnrichmentHook(() -> root, states); - hook.finallyAfter( - ctx("flag", "user-1"), - details("flag", "on", "v", metadata(42, true)), - Collections.emptyMap()); - final SpanEnrichmentAccumulator state = states.peek(root); - assertTrue(state != null && state.serialIdsView().contains(42)); - assertEquals(1, state.subjectCount(), "doLog=true + targeting key => subject recorded"); + assertEquals(1, captured.size()); + final SpanEnrichmentEvent event = captured.get(0); + assertTrue(event.hasSerialId()); + assertEquals(7, event.serialId()); + assertFalse( + event.doLog(), "doLog=false must be carried through so the write side skips subject"); } - // ---- 3. finished-root (accumulate + dedupe, then flush via interceptor) ---- - @Test - void finishedRootFlushesFlagsEncTag() { - final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final AgentSpan root = rootSpan(); - // accumulate {100,108,128,130} with a duplicate 100 to prove dedupe - capture(hook, root, "f1", "user-1", "on", "v", 100, false); - capture(hook, root, "f2", "user-1", "on", "v", 108, false); - capture(hook, root, "f3", "user-1", "on", "v", 128, false); - capture(hook, root, "f4", "user-1", "on", "v", 130, false); - capture(hook, root, "f1", "user-1", "on", "v", 100, false); // dup - - boundInterceptor(states).onTraceComplete(Collections.singletonList(root)); - - verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZAgUAg=="); - // state cleared after flush - assertTrue(states.isEmpty(), "state must be cleared on flush"); + void malformedSerialIdDispatchesNothing() { + final ImmutableMetadata bad = + ImmutableMetadata.builder() + .addString(SpanEnrichmentHook.METADATA_SERIAL_ID, "not-a-number") + .addString(SpanEnrichmentHook.METADATA_DO_LOG, "true") + .build(); + new SpanEnrichmentHook() + .finallyAfter( + ctx("flag", "user-1"), details("flag", "on", "v", bad), Collections.emptyMap()); + assertTrue(captured.isEmpty(), "a malformed serial id must never break eval or dispatch"); } - // ---- 4. error/default variant (missing variant -> ffe_runtime_defaults JSON object) ---- + // ---- runtime-default branch (missing variant) ---- @Test - void runtimeDefaultMissingVariantWritesJsonObject() { - final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final AgentSpan root = rootSpan(); - // no serial id + null variant => runtime default; object value must be JSON-stringified + void missingVariantDispatchesRuntimeDefaultWithNativeMap() { final Map objectValue = Collections.singletonMap("k", "val"); - hook.capture( - root, - ctx("obj-flag", "user-1"), - FlagEvaluationDetails.builder() - .flagKey("obj-flag") - .variant(null) - .value(objectValue) - .flagMetadata(ImmutableMetadata.builder().build()) - .build()); - - boundInterceptor(states).onTraceComplete(Collections.singletonList(root)); - - // ffe_runtime_defaults is a JSON object string, NOT [object Object]/toString. - verify(root) - .setTag( - SpanEnrichmentAccumulator.TAG_RUNTIME_DEFAULTS, - "{\"obj-flag\":\"{\\\"k\\\":\\\"val\\\"}\"}"); - // no flags tag when there are no serial ids - verify(root, never()).setTag(eq(SpanEnrichmentAccumulator.TAG_FLAGS_ENC), anyString()); + new SpanEnrichmentHook() + .finallyAfter( + ctx("obj-flag", "user-1"), + details("obj-flag", null, objectValue, ImmutableMetadata.builder().build()), + Collections.emptyMap()); + + assertEquals(1, captured.size()); + final SpanEnrichmentEvent event = captured.get(0); + assertFalse(event.hasSerialId()); + assertEquals("obj-flag", event.flagKey()); + assertEquals( + objectValue, event.defaultValue(), "a native map default passes through unchanged"); } - // ---- 4b. real OpenFeature object path: a Value structure default must serialize to JSON ---- - - /** - * The real object-evaluation path hands the runtime default in as a {@code - * dev.openfeature.sdk.Value}, not a raw {@code Map}. The accumulator must unwrap the {@code - * Value} and emit JSON (matching Node's {@code JSON.stringify}), never {@code Value.toString()} - * (which is {@code "Value(innerObject=...)"}). - */ @Test - void runtimeDefaultStructureValueSerializesAsJsonNotToString() { - final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final AgentSpan root = rootSpan(); - - // Single-key structure so the exact-string assertion does not depend on the OpenFeature SDK's - // internal key ordering. (The multi-key / nesting behaviour is covered by the direct - // stringifyDefault assertions below.) + void missingVariantUnwrapsOpenFeatureValueStructureToNativeMap() { final Map inner = new LinkedHashMap<>(); inner.put("enabled", new Value(true)); final Value structureDefault = new Value(new ImmutableStructure(inner)); - hook.capture( - root, - ctx("struct-flag", "user-1"), - FlagEvaluationDetails.builder() - .flagKey("struct-flag") - .variant(null) - .value(structureDefault) - .flagMetadata(ImmutableMetadata.builder().build()) - .build()); - - boundInterceptor(states).onTraceComplete(Collections.singletonList(root)); - - // {"struct-flag":"{\"enabled\":true}"} — note: NO "Value(innerObject=...)". - verify(root) - .setTag( - SpanEnrichmentAccumulator.TAG_RUNTIME_DEFAULTS, - "{\"struct-flag\":\"{\\\"enabled\\\":true}\"}"); + new SpanEnrichmentHook() + .finallyAfter( + ctx("struct-flag", "user-1"), + details("struct-flag", null, structureDefault, ImmutableMetadata.builder().build()), + Collections.emptyMap()); + + assertEquals(1, captured.size()); + final Object value = captured.get(0).defaultValue(); + // Crucially a native Map, NOT an OpenFeature Value — the seam carries only JDK types. + final Map asMap = assertInstanceOf(Map.class, value); + assertEquals(Boolean.TRUE, asMap.get("enabled")); } - /** - * Direct stringify of a structured {@code Value}: asserts the value is JSON (objects + nested - * scalars), not {@code Value.toString()}. The structure is single-key to keep ordering - * deterministic across OpenFeature SDK versions. - */ + // ---- unwrapDefaultValue (the Value -> native conversion) ---- + @Test - void stringifyDefaultUnwrapsValueStructureToJson() { - final Map inner = new LinkedHashMap<>(); - inner.put("count", new Value(42)); - final Value structureDefault = new Value(new ImmutableStructure(inner)); - assertEquals("{\"count\":42}", SpanEnrichmentAccumulator.stringifyDefault(structureDefault)); + void unwrapConvertsValueScalarsToNative() { + assertEquals("hello", SpanEnrichmentHook.unwrapDefaultValue(new Value("hello"))); + assertEquals(Boolean.TRUE, SpanEnrichmentHook.unwrapDefaultValue(new Value(true))); + assertEquals(7, SpanEnrichmentHook.unwrapDefaultValue(new Value(7))); + assertNull(SpanEnrichmentHook.unwrapDefaultValue(new Value())); } - /** - * A list-valued {@code Value} default serializes to a JSON array, with nested Values unwrapped. - */ @Test - void runtimeDefaultListValueSerializesAsJsonArray() { + void unwrapConvertsValueListToNativeList() { final Value listDefault = new Value(Arrays.asList(new Value("a"), new Value(2), new Value(true))); - // direct stringify assertion (exact bytes) - assertEquals("[\"a\",2,true]", SpanEnrichmentAccumulator.stringifyDefault(listDefault)); - } - - /** Scalar Values collapse to the same string form Node's String(value) produces. */ - @Test - void scalarValueDefaultsMatchNodeStringForm() { - assertEquals("hello", SpanEnrichmentAccumulator.stringifyDefault(new Value("hello"))); - assertEquals("true", SpanEnrichmentAccumulator.stringifyDefault(new Value(true))); - assertEquals("7", SpanEnrichmentAccumulator.stringifyDefault(new Value(7))); - assertEquals("null", SpanEnrichmentAccumulator.stringifyDefault(new Value())); + final Object out = SpanEnrichmentHook.unwrapDefaultValue(listDefault); + assertEquals(Arrays.asList("a", 2, Boolean.TRUE), out); } - // ---- 5. per-subject cap (10 subjects / 20 experiments / doLog gating) ---- - @Test - void subjectCapsAndDoLogGating() { - final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); - - // doLog gating: addSubject only happens when doLog true (driven via the hook branch below). - final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final AgentSpan root = rootSpan(); - capture(hook, root, "f", "user-A", "on", "v", 1, false); // doLog=false => no subject - assertEquals(0, states.peek(root).subjectCount(), "doLog=false must not record a subject"); - capture(hook, root, "f", "user-A", "on", "v", 2, true); // doLog=true => subject recorded - assertEquals(1, states.peek(root).subjectCount()); - - // per-subject experiment cap: 20 max - for (int i = 0; i < 25; i++) { - acc.addSubject("subjectX", i); - } - final Map tags = acc.toSpanTags(); - final SortedSet decoded = - decodeDeltaVarint( - // ffe_subjects_enc is {"":""}; extract the single base64 value - tags.get(SpanEnrichmentAccumulator.TAG_SUBJECTS_ENC) - .replaceAll("^\\{\"[a-f0-9]+\":\"", "") - .replaceAll("\"\\}$", "")); - assertEquals(SpanEnrichmentAccumulator.MAX_EXPERIMENTS_PER_SUBJECT, decoded.size()); - - // subject cap: 10 max distinct subjects - final SpanEnrichmentAccumulator acc2 = new SpanEnrichmentAccumulator(); - for (int i = 0; i < 15; i++) { - acc2.addSubject("subject-" + i, i); - } - assertEquals(SpanEnrichmentAccumulator.MAX_SUBJECTS, acc2.subjectCount()); + void unwrapPassesNativeValuesThrough() { + final Map native0 = Collections.singletonMap("a", "b"); + assertEquals(native0, SpanEnrichmentHook.unwrapDefaultValue(native0)); + assertEquals("plain", SpanEnrichmentHook.unwrapDefaultValue("plain")); + assertNull(SpanEnrichmentHook.unwrapDefaultValue(null)); } - // ---- 6. max-200 serial ids ---- + // ---- error isolation ---- @Test - void max200SerialIdsEnforced() { - final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); - for (int i = 0; i < 300; i++) { - acc.addSerialId(i); - } - assertEquals( - SpanEnrichmentAccumulator.MAX_SERIAL_IDS, - acc.serialIdsView().size(), - "serial ids must be capped at 200"); + void nullDetailsDispatchesNothing() { + new SpanEnrichmentHook().finallyAfter(null, null, null); + assertTrue(captured.isEmpty()); } - // ---- 7. JSON/object-default (json not toString + 64-char truncation) ---- + // ---- Provider gating ---- @Test - void objectDefaultJsonAndTruncation() { - // object -> JSON, not toString - assertEquals( - "{\"a\":\"b\"}", - SpanEnrichmentAccumulator.stringifyDefault(Collections.singletonMap("a", "b"))); - // scalar string -> as-is - assertEquals("hello", SpanEnrichmentAccumulator.stringifyDefault("hello")); - // null -> "null" - assertEquals("null", SpanEnrichmentAccumulator.stringifyDefault(null)); - - // 64-char truncation (first-wins handled separately) - final StringBuilder longValue = new StringBuilder(); - for (int i = 0; i < 100; i++) { - longValue.append('x'); - } - final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); - acc.addDefault("flag", longValue.toString()); - final String tag = acc.toSpanTags().get(SpanEnrichmentAccumulator.TAG_RUNTIME_DEFAULTS); - // {"flag":"<64 x's>"} - final String expectedValue = - longValue.substring(0, SpanEnrichmentAccumulator.MAX_DEFAULT_VALUE_LENGTH); - assertEquals("{\"flag\":\"" + expectedValue + "\"}", tag); - - // first-wins: a second addDefault for the same flag is ignored - acc.addDefault("flag", "second"); - assertEquals(1, acc.defaultCount()); - } - - // ---- gate-off negative control (no ffe_*, no hook, no state) ---- - - @Test - void gateOffConstructsNothingAndAccumulatesNoState() { - // Gate OFF via the injectable override (no static config mocking). + void gateOffConstructsNoHook() { final Provider provider = new Provider(new Provider.Options(), null, Boolean.FALSE); - - // No hook, no state store constructed (zero idle overhead). assertNull(provider.spanEnrichmentHook(), "gate off => no span-enrichment hook"); - assertNull(provider.spanEnrichmentStates(), "gate off => no span-enrichment state store"); - // getProviderHooks must not contain a SpanEnrichmentHook. - final List hooks = provider.getProviderHooks(); - for (final Hook hook : hooks) { + for (final Hook hook : provider.getProviderHooks()) { assertFalse( hook instanceof SpanEnrichmentHook, "gate off => SpanEnrichmentHook never registered"); } } - /** - * getProviderHooks() is called on every evaluation; it must return the SAME precomputed list - * (allocating nothing) regardless of gate state. - */ - @Test - void getProviderHooksReturnsSameInstanceEachCall() { - final Provider gateOff = new Provider(new Provider.Options(), null, Boolean.FALSE); - assertTrue( - gateOff.getProviderHooks() == gateOff.getProviderHooks(), - "gate off => getProviderHooks allocates nothing (same instance)"); - - final Provider gateOn = - new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); - assertTrue( - gateOn.getProviderHooks() == gateOn.getProviderHooks(), - "gate on => getProviderHooks allocates nothing (same instance)"); - } - - // ---- gate-on construction + provider-close cleanup ---- - @Test - void gateOnConstructsHookAndStateThenShutdownUnbinds() { - final Provider provider = - new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); + void gateOnConstructsHookAndRegistersItInProviderHooks() { + final Provider provider = new Provider(new Provider.Options(), null, Boolean.TRUE); assertTrue(provider.spanEnrichmentHook() != null, "gate on => hook constructed"); - assertTrue(provider.spanEnrichmentStates() != null, "gate on => state store constructed"); - // the process-wide interceptor is bound to this provider's store - assertTrue( - SpanEnrichmentInterceptor.INSTANCE.activeStates() == provider.spanEnrichmentStates(), - "gate on => interceptor bound to this provider's store"); - // hook registered in provider hooks boolean registered = false; for (final Hook hook : provider.getProviderHooks()) { if (hook instanceof SpanEnrichmentHook) { @@ -456,48 +226,18 @@ void gateOnConstructsHookAndStateThenShutdownUnbinds() { } } assertTrue(registered, "gate on => SpanEnrichmentHook registered in getProviderHooks"); - - // provider close unbinds + drains ITS OWN state. - final SpanEnrichmentStates providerStates = provider.spanEnrichmentStates(); - providerStates.getOrCreate(mock(AgentSpan.class)); - assertFalse(providerStates.isEmpty()); - provider.shutdown(); - assertNull( - SpanEnrichmentInterceptor.INSTANCE.activeStates(), "shutdown unbinds the active store"); - assertTrue(providerStates.isEmpty(), "shutdown drains residual state"); } - // ---- error isolation: enrichment never throws ---- - @Test - void captureNeverThrowsOnNullInputs() { - final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - // null details handled in finallyAfter; capture with empty metadata + null variant - hook.finallyAfter(null, null, null); - assertTrue(states.isEmpty()); - } - - // ---- interceptor inert when unbound / empty trace robustness ---- - - @Test - void interceptorNoOpsWhenUnboundOrEmpty() { - // empty trace - SpanEnrichmentInterceptor.INSTANCE.bind(states); + void getProviderHooksReturnsSameInstanceEachCall() { + final Provider gateOff = new Provider(new Provider.Options(), null, Boolean.FALSE); assertTrue( - SpanEnrichmentInterceptor.INSTANCE.onTraceComplete(Collections.emptyList()).isEmpty()); - // unbound (inert) - SpanEnrichmentInterceptor.INSTANCE.unbind(states); - final AgentSpan root = mock(AgentSpan.class); - final List trace = Collections.singletonList(root); - SpanEnrichmentInterceptor.INSTANCE.onTraceComplete(trace); - verify(root, never()).setTag(anyString(), anyString()); - } - - // ---- interceptor priority uniqueness ---- + gateOff.getProviderHooks() == gateOff.getProviderHooks(), + "gate off => getProviderHooks allocates nothing (same instance)"); - @Test - void interceptorPriorityIsUnique() { - // Distinct from AbstractTraceInterceptor.Priority values (0,1,2,3, MAX-2, MAX-1, MAX). - assertEquals(4, SpanEnrichmentInterceptor.INSTANCE.priority()); + final Provider gateOn = new Provider(new Provider.Options(), null, Boolean.TRUE); + assertTrue( + gateOn.getProviderHooks() == gateOn.getProviderHooks(), + "gate on => getProviderHooks allocates nothing (same instance)"); } } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java deleted file mode 100644 index d3a827617c2..00000000000 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentLifecycleRegressionTest.java +++ /dev/null @@ -1,411 +0,0 @@ -package datadog.trace.api.openfeature; - -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import datadog.trace.api.interceptor.MutableSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import dev.openfeature.sdk.EvaluationContext; -import dev.openfeature.sdk.FlagEvaluationDetails; -import dev.openfeature.sdk.FlagValueType; -import dev.openfeature.sdk.HookContext; -import dev.openfeature.sdk.ImmutableContext; -import dev.openfeature.sdk.ImmutableMetadata; -import java.lang.ref.WeakReference; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -/** - * Regression suite for the per-root-span lifecycle and reconfiguration behaviour of span - * enrichment. Each test exercises a scenario the single-root, single-thread, single-provider happy - * path does not cover. - * - *
      - *
    • partial flush — a fragment of child spans only (the still-open root is excluded by - * dd-trace-core) must NOT drain the accumulator; pre-flush flags must survive onto the root - * at final completion. - *
    • unbounded leak — traces that never reach the interceptor (dropped / Noop / never-finishing) - * must not leak accumulator entries unboundedly; the store is weak-keyed by the local-root - * span, so entries are collected once their root becomes unreachable (no cap / eviction). - *
    • reconfiguration — after a provider closes, a new gate-on provider must still enrich (the - * process-wide interceptor rebinds), and a closing provider must never clobber a newer - * provider's in-flight state. - *
    • distinct traces — two distinct traces (distinct local-root span objects) must not merge - * enrichment state. - *
    • keying symmetry — the root captured by the hook and the root resolved by the interceptor - * from the flushed fragment must be the SAME object, so identity keying round-trips. - *
    - */ -class SpanEnrichmentLifecycleRegressionTest { - - @AfterEach - void resetInterceptor() { - // The interceptor is a process-wide singleton; leave it inert for the next test. - SpanEnrichmentInterceptor.INSTANCE.unbind(SpanEnrichmentInterceptor.INSTANCE.activeStates()); - } - - // ---- helpers ---- - - private static ImmutableMetadata serialMeta(final int serialId, final boolean doLog) { - return ImmutableMetadata.builder() - .addString(SpanEnrichmentHook.METADATA_SERIAL_ID, Integer.toString(serialId)) - .addString(SpanEnrichmentHook.METADATA_DO_LOG, String.valueOf(doLog)) - .build(); - } - - private static FlagEvaluationDetails serialDetails( - final String flagKey, final int serialId, final boolean doLog) { - return FlagEvaluationDetails.builder() - .flagKey(flagKey) - .variant("on") - .value("v") - .flagMetadata(serialMeta(serialId, doLog)) - .build(); - } - - private static HookContext ctx(final String flagKey, final String targetingKey) { - final EvaluationContext ec = - targetingKey == null ? new ImmutableContext() : new ImmutableContext(targetingKey); - return HookContext.from(flagKey, FlagValueType.STRING, null, null, ec, "default"); - } - - /** A mock child span whose local root is {@code root} but which is itself NOT the root. */ - private static AgentSpan childOf(final AgentSpan root) { - final AgentSpan child = mock(AgentSpan.class); - when(child.getLocalRootSpan()).thenReturn(root); - return child; - } - - /** A mock local-root span reporting itself as its own local root. */ - private static AgentSpan rootSpan() { - final AgentSpan root = mock(AgentSpan.class); - when(root.getLocalRootSpan()).thenReturn(root); - return root; - } - - // ===================================================================================== - // partial flush must not drop captured state or misattribute tags - // ===================================================================================== - - /** - * Models the real dd-trace-core sequence for a long-running trace: - * - *
      - *
    1. flags are evaluated (captured into the accumulator), - *
    2. a PARTIAL flush fires {@code onTraceComplete} with a fragment of CHILD spans only — the - * still-open local root is excluded (held back via {@code rootSpanWritten}), - *
    3. more flags are evaluated, - *
    4. the FINAL flush fires {@code onTraceComplete} with the root present. - *
    - * - * All serial ids — both pre- and post-partial-flush — must appear on the root's {@code - * ffe_flags_enc} tag, and nothing must be written on the partial flush. - */ - @Test - void partialFlushExcludingRootPreservesPreFlushFlags() { - final SpanEnrichmentStates states = new SpanEnrichmentStates(); - final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final SpanEnrichmentInterceptor interceptor = SpanEnrichmentInterceptor.INSTANCE; - interceptor.bind(states); - - final AgentSpan root = rootSpan(); - - // (1) pre-partial-flush evaluations - hook.capture(root, ctx("f1", "user-1"), serialDetails("f1", 100, false)); - hook.capture(root, ctx("f2", "user-1"), serialDetails("f2", 108, false)); - - // (2) PARTIAL flush: a fragment of children only — the open root is NOT in the collection. - final AgentSpan child1 = childOf(root); - final AgentSpan child2 = childOf(root); - final List partialFragment = Arrays.asList(child1, child2); - interceptor.onTraceComplete(partialFragment); - - // No tags may be written on a partial flush, and the accumulator must survive intact. - verify(child1, never()).setTag(anyString(), anyString()); - verify(child2, never()).setTag(anyString(), anyString()); - verify(root, never()).setTag(anyString(), anyString()); - final SpanEnrichmentAccumulator surviving = states.peek(root); - assertNotNull(surviving, "partial flush must NOT remove the accumulator"); - assertTrue( - surviving.serialIdsView().contains(100) && surviving.serialIdsView().contains(108), - "pre-flush serial ids must survive a partial flush"); - - // (3) more evaluations after the partial flush - hook.capture(root, ctx("f3", "user-1"), serialDetails("f3", 128, false)); - hook.capture(root, ctx("f4", "user-1"), serialDetails("f4", 130, false)); - - // (4) FINAL flush: the root is present in the fragment (alongside a late child). - final AgentSpan lateChild = childOf(root); - interceptor.onTraceComplete(Arrays.asList(lateChild, root)); - - // The full set {100,108,128,130} -> golden "ZAgUAg==" must land on the root. - verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZAgUAg=="); - assertTrue(states.isEmpty(), "state must be removed only on the final flush"); - } - - /** - * A partial flush whose first span reports a non-null local root that is absent from the fragment - * must be treated as "not the final write" and leave state intact — even when there are several - * children. Guards the "never tag a non-root span" concern. - */ - @Test - void partialFlushNeverWritesTagsOnAChildSpan() { - final SpanEnrichmentStates states = new SpanEnrichmentStates(); - final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final SpanEnrichmentInterceptor interceptor = SpanEnrichmentInterceptor.INSTANCE; - interceptor.bind(states); - - final AgentSpan root = rootSpan(); - hook.capture(root, ctx("f", "user-1"), serialDetails("f", 7, false)); - - final AgentSpan child = childOf(root); - interceptor.onTraceComplete(Collections.singletonList(child)); - - verify(child, never()).setTag(anyString(), anyString()); - verify(root, never()).setTag(anyString(), anyString()); - assertNotNull(states.peek(root), "child-only fragment must keep state"); - } - - // ===================================================================================== - // never-completing traces must not leak accumulator entries unboundedly - // ===================================================================================== - - /** - * There is no cap and no eviction: while their local-root spans are still reachable, every - * never-completing trace keeps its own accumulator. The weak-keyed store bounds memory by - * reachability (see {@link #unreachableRootStateIsWeaklyCollected()}), not by a fixed count, so - * inserting far more distinct roots than the old 4096 cap never drops a still-live entry. - */ - @Test - void neverCompletingTracesAreNotCapped() { - final SpanEnrichmentStates states = new SpanEnrichmentStates(); - final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - - final int churn = 20_000; // well beyond the old 4096 cap - // Hold strong references so nothing is GC'd during the test — proves there is no size cap. - final List liveRoots = new ArrayList<>(churn); - for (int i = 0; i < churn; i++) { - final AgentSpan root = rootSpan(); - liveRoots.add(root); - hook.capture(root, ctx("f", "user-" + i), serialDetails("f", i % 1000 + 1, false)); - } - assertEquals(churn, states.size(), "no cap/eviction: every live root keeps its accumulator"); - } - - /** - * Weak-reference semantics: once a never-completing trace's local-root span becomes unreachable, - * its accumulator entry is collected — the store cannot leak unboundedly without any cap. GC is - * non-deterministic, so this polls with explicit {@code System.gc()} hints until the entry - * clears. - */ - @Test - void unreachableRootStateIsWeaklyCollected() { - final SpanEnrichmentStates states = new SpanEnrichmentStates(); - final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - - AgentSpan root = rootSpan(); - hook.capture(root, ctx("f", "user-1"), serialDetails("f", 1, false)); - assertEquals(1, states.size(), "one in-flight accumulator before the root is dropped"); - - final WeakReference ref = new WeakReference<>(root); - root = null; // drop the only strong reference to the never-completing root - - await() - .atMost(Duration.ofSeconds(10)) - .pollInterval(Duration.ofMillis(50)) - .untilAsserted( - () -> { - System.gc(); - assertNull(ref.get(), "unreferenced root must be collectable"); - assertEquals(0, states.size(), "accumulator must be purged with its collected root"); - }); - } - - // ===================================================================================== - // reconfiguration: a closing provider must not permanently disable enrichment, nor - // clobber a newer provider's state - // ===================================================================================== - - /** - * The core reconfiguration regression: a first gate-on provider shuts down, then a second gate-on - * provider is created. Enrichment must still work for the second provider. The process-wide - * interceptor is registered once and rebound, so the second provider is NOT rejected as a - * duplicate (which would permanently disable enrichment). - */ - @Test - void newProviderAfterShutdownStillEnriches() { - // First provider registers and binds. - final Provider first = - new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); - assertNotNull(first.spanEnrichmentStates(), "first provider wires a store"); - first.shutdown(); - assertNull( - SpanEnrichmentInterceptor.INSTANCE.activeStates(), - "first provider's shutdown leaves the interceptor inert"); - - // Second provider is created AFTER the first closed. With the old per-provider interceptor - // model - // this registration would be rejected as a duplicate and enrichment would be permanently off. - final Provider second = - new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); - final SpanEnrichmentStates secondStates = second.spanEnrichmentStates(); - assertNotNull(secondStates, "second provider wires a store"); - assertTrue( - SpanEnrichmentInterceptor.INSTANCE.activeStates() == secondStates, - "the interceptor must rebind to the second provider's store"); - - // End-to-end: the second provider captures and the interceptor flushes onto the root. - final AgentSpan root = rootSpan(); - second.spanEnrichmentHook().capture(root, ctx("f", "user-1"), serialDetails("f", 5, false)); - SpanEnrichmentInterceptor.INSTANCE.onTraceComplete(Collections.singletonList(root)); - verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "BQ=="); // {5} -> 0x05 - } - - /** - * Overlapping reconfiguration: a second provider rebinds the interceptor while the first is still - * "open"; when the FIRST provider later shuts down, it must NOT clear the second provider's - * in-flight state (its unbind is a no-op because it is no longer the active provider). - */ - @Test - void lateShutdownOfDisplacedProviderDoesNotClobberActiveProvider() { - final Provider first = - new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); - final Provider second = - new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); - final SpanEnrichmentStates secondStates = second.spanEnrichmentStates(); - - // Second provider is now the active one and has live state. - final AgentSpan root = rootSpan(); - second.spanEnrichmentHook().capture(root, ctx("f", "user-1"), serialDetails("f", 9, false)); - assertNotNull(secondStates.peek(root), "second provider has in-flight state"); - assertTrue(SpanEnrichmentInterceptor.INSTANCE.activeStates() == secondStates); - - // The DISPLACED first provider shuts down late. The active (second) provider must be untouched. - first.shutdown(); - assertTrue( - SpanEnrichmentInterceptor.INSTANCE.activeStates() == secondStates, - "late shutdown of a displaced provider must not unbind the active provider"); - assertNotNull( - secondStates.peek(root), - "late shutdown of a displaced provider must not clear the active provider's state"); - - // Sanity: the second provider still flushes correctly. - SpanEnrichmentInterceptor.INSTANCE.onTraceComplete(Collections.singletonList(root)); - verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "CQ=="); // {9} -> 0x09 - } - - /** Each provider owns a DISTINCT state store, so they never share mutable state. */ - @Test - void eachProviderOwnsADistinctStateStore() { - final Provider a = - new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); - final Provider b = - new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); - assertNotNull(a.spanEnrichmentStates()); - assertNotNull(b.spanEnrichmentStates()); - assertTrue( - a.spanEnrichmentStates() != b.spanEnrichmentStates(), - "providers must own distinct state stores"); - - a.spanEnrichmentStates().getOrCreate(rootSpan()); - assertTrue(b.spanEnrichmentStates().isEmpty(), "stores are isolated"); - } - - /** - * Within ONE provider, the capture hook and the bound interceptor must share the SAME store, so a - * capture is visible to the flush. - */ - @Test - void hookAndInterceptorOfOneProviderShareTheSameStore() { - final Provider provider = - new Provider(new Provider.Options(), null, Boolean.TRUE, interceptor -> true); - final AgentSpan root = rootSpan(); - provider.spanEnrichmentHook().capture(root, ctx("f", "user-1"), serialDetails("f", 9, false)); - assertNotNull( - SpanEnrichmentInterceptor.INSTANCE.activeStates().peek(root), - "hook capture must be visible in the bound store (same instance)"); - } - - // ===================================================================================== - // distinct traces (distinct local-root span objects) must not merge - // ===================================================================================== - - /** - * Two distinct traces have two distinct local-root span objects, so identity keying keeps their - * accumulators SEPARATE regardless of any trace-id bits. (Under the old hex-string keying this - * guarded against 128-bit ids sharing their low-order 64 bits; identity keying makes it - * inherent.) - */ - @Test - void distinctTracesDoNotMerge() { - final SpanEnrichmentStates states = new SpanEnrichmentStates(); - final SpanEnrichmentHook hook = new SpanEnrichmentHook(states); - final SpanEnrichmentInterceptor interceptor = SpanEnrichmentInterceptor.INSTANCE; - interceptor.bind(states); - - final AgentSpan rootA = rootSpan(); - final AgentSpan rootB = rootSpan(); - - // Capture distinct serial ids under each distinct root. - hook.capture(rootA, ctx("fa", "user-A"), serialDetails("fa", 100, false)); - hook.capture(rootB, ctx("fb", "user-B"), serialDetails("fb", 130, false)); - - // They must NOT have merged into one accumulator. - assertEquals(2, states.size(), "distinct root spans must not share an accumulator"); - - // Flush trace A: only {100} (not {100,130}). - interceptor.onTraceComplete(Collections.singletonList(rootA)); - verify(rootA).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZA=="); // {100} -> 0x64 - - // Flush trace B: only {130}. - interceptor.onTraceComplete(Collections.singletonList(rootB)); - verify(rootB).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ggE="); // {130} -> 0x82 0x01 - } - - // ===================================================================================== - // keying symmetry: the hook's captured root and the interceptor's resolved root are identical - // ===================================================================================== - - /** - * The whole identity-keying design rests on the hook and the interceptor referencing the SAME - * local-root span object. The hook captures against the root resolved by its {@code - * RootSpanResolver}; the interceptor resolves the root from the flushed fragment via {@code - * findLocalRootInFragment}. This asserts both resolve to the same instance so the accumulator - * captured on the eval side is exactly the one removed and flushed on the write side. - */ - @Test - void hookCapturedRootAndInterceptorResolvedRootAreSameObject() { - final SpanEnrichmentStates states = new SpanEnrichmentStates(); - final AgentSpan root = rootSpan(); - // The hook resolves its root via the injected resolver (the production DEFAULT_RESOLVER calls - // activeSpan().getLocalRootSpan(); here we inject the same object the fragment will report). - final SpanEnrichmentHook hook = new SpanEnrichmentHook(() -> root, states); - final SpanEnrichmentInterceptor interceptor = SpanEnrichmentInterceptor.INSTANCE; - interceptor.bind(states); - - hook.finallyAfter( - ctx("f", "user-1"), serialDetails("f", 5, false), Collections.emptyMap()); - final SpanEnrichmentAccumulator captured = states.peek(root); - assertNotNull(captured, "hook captures against the resolved local root"); - - // The interceptor resolves the root from the fragment; it must be the same object, so the - // remove hits the captured accumulator and flushes it onto that exact root. - final AgentSpan child = childOf(root); - interceptor.onTraceComplete(Arrays.asList(child, root)); - verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "BQ=="); // {5} -> 0x05 - assertTrue(states.isEmpty(), "the interceptor removed the same accumulator the hook created"); - } -} diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java index b9d73ffa7ab..2704b4be341 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java @@ -13,8 +13,12 @@ public interface ConfigListener extends Consumer {} public interface ExposureListener extends Consumer {} + public interface SpanEnrichmentListener extends Consumer {} + private static final List CONFIG_LISTENERS = new CopyOnWriteArrayList<>(); private static final List EXPOSURE_LISTENERS = new CopyOnWriteArrayList<>(); + private static final List SPAN_ENRICHMENT_LISTENERS = + new CopyOnWriteArrayList<>(); private static final AtomicReference CURRENT_CONFIG = new AtomicReference<>(); @@ -49,4 +53,16 @@ public static void removeExposureListener(final ExposureListener listener) { public static void dispatch(final ExposureEvent event) { EXPOSURE_LISTENERS.forEach(listener -> listener.accept(event)); } + + public static void addSpanEnrichmentListener(final SpanEnrichmentListener listener) { + SPAN_ENRICHMENT_LISTENERS.add(listener); + } + + public static void removeSpanEnrichmentListener(final SpanEnrichmentListener listener) { + SPAN_ENRICHMENT_LISTENERS.remove(listener); + } + + public static void dispatch(final SpanEnrichmentEvent event) { + SPAN_ENRICHMENT_LISTENERS.forEach(listener -> listener.accept(event)); + } } diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/SpanEnrichmentEvent.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/SpanEnrichmentEvent.java new file mode 100644 index 00000000000..a7518c71743 --- /dev/null +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/SpanEnrichmentEvent.java @@ -0,0 +1,82 @@ +package datadog.trace.api.featureflag; + +/** + * Bootstrap-classloader carrier for a single feature-flag evaluation that must be reflected onto + * the local-root APM span (span enrichment). It crosses the app-classloader → agent-classloader + * boundary through {@link FeatureFlaggingGateway}, so it holds only JDK types — never an + * OpenFeature or tracer type — exactly like the sibling exposure/config payloads. + * + *

    The capture side (the published {@code dd-openfeature} provider) decides which of the two + * shapes applies and unwraps any OpenFeature {@code Value} to its native Java form before + * dispatching; the write side (agent-side listener) resolves the active local root and accumulates. + * + *

      + *
    • serial-id ({@link #serialId(int, boolean, String)}) — a UFC split with a serial id, + * plus the {@code doLog} flag and the (optional) targeting key used to record the subject. + *
    • runtime-default ({@link #runtimeDefault(String, Object)}) — a flag that resolved to + * its runtime default (missing variant); carries the flag key and the native default value. + *
    + */ +public final class SpanEnrichmentEvent { + + private final boolean serialIdPresent; + private final int serialId; + private final boolean doLog; + private final String targetingKey; + private final String flagKey; + private final Object defaultValue; + + private SpanEnrichmentEvent( + final boolean serialIdPresent, + final int serialId, + final boolean doLog, + final String targetingKey, + final String flagKey, + final Object defaultValue) { + this.serialIdPresent = serialIdPresent; + this.serialId = serialId; + this.doLog = doLog; + this.targetingKey = targetingKey; + this.flagKey = flagKey; + this.defaultValue = defaultValue; + } + + /** A UFC split evaluation carrying a serial id (and, when {@code doLog}, a subject). */ + public static SpanEnrichmentEvent serialId( + final int serialId, final boolean doLog, final String targetingKey) { + return new SpanEnrichmentEvent(true, serialId, doLog, targetingKey, null, null); + } + + /** + * A runtime-default evaluation (missing variant). {@code value} must already be unwrapped to a + * native Java type (Map/List/scalar/null) by the caller. + */ + public static SpanEnrichmentEvent runtimeDefault(final String flagKey, final Object value) { + return new SpanEnrichmentEvent(false, 0, false, null, flagKey, value); + } + + /** True for the serial-id shape; false for the runtime-default shape. */ + public boolean hasSerialId() { + return serialIdPresent; + } + + public int serialId() { + return serialId; + } + + public boolean doLog() { + return doLog; + } + + public String targetingKey() { + return targetingKey; + } + + public String flagKey() { + return flagKey; + } + + public Object defaultValue() { + return defaultValue; + } +} diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 3291e239d40..584e867f321 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -22,9 +22,14 @@ dependencies { api(project(":utils:queue-utils")) compileOnly(project(":dd-trace-core")) // shading does not work with this one + // Span-enrichment write tier: TraceInterceptor / GlobalTracer / AgentTracer / AgentSpan. This is + // an agent-side module (shaded into the agent, never published), so depending on the tracer's + // internal API is expected — unlike the published dd-openfeature product API. + compileOnly(project(":internal-api")) testImplementation(libs.bundles.junit5) testImplementation(libs.bundles.mockito) + testImplementation(project(":internal-api")) testImplementation(project(":utils:test-utils")) testImplementation(project(":dd-java-agent:testing")) } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentAccumulator.java similarity index 69% rename from products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java rename to products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentAccumulator.java index 594650ac08c..1b90b3d265f 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentAccumulator.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentAccumulator.java @@ -1,8 +1,5 @@ -package datadog.trace.api.openfeature; +package com.datadog.featureflag; -import dev.openfeature.sdk.Structure; -import dev.openfeature.sdk.Value; -import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; @@ -17,9 +14,14 @@ * ULeb128Encoder}. * *

    Instances are created lazily and held in a {@link SpanEnrichmentStates} store, keyed by the - * local-root span's full trace id. The capture hook ({@link SpanEnrichmentHook}) writes; the write - * interceptor ({@link SpanEnrichmentInterceptor}) reads and clears. When the span-enrichment gate - * is off, no store and no accumulator are ever created, so there is no idle per-span overhead. + * local-root span object. The agent-side {@link SpanEnrichmentWriter} writes (from the flag-eval + * seam); the write interceptor ({@link SpanEnrichmentInterceptor}) reads and clears. When the + * span-enrichment gate is off, no seam events are dispatched, so no store and no accumulator are + * ever created and there is no idle per-span overhead. + * + *

    Runtime-default values arrive already unwrapped to native Java types (the capture side unwraps + * any OpenFeature {@code Value} before crossing the seam), so this class has no OpenFeature + * dependency. * *

    Output tag shapes: * @@ -83,8 +85,9 @@ synchronized void addSubject(final String targetingKey, final int id) { } /** - * Records a runtime-default value for {@code flagKey} (first-wins). Object values are serialized - * to JSON (NOT {@code toString()}); the result is truncated to {@link #MAX_DEFAULT_VALUE_LENGTH}. + * Records a runtime-default value for {@code flagKey} (first-wins). Structured values (Map/List) + * are serialized to JSON (NOT {@code toString()}); the result is truncated to {@link + * #MAX_DEFAULT_VALUE_LENGTH}. */ synchronized void addDefault(final String flagKey, final Object value) { if (flagKey == null) { @@ -142,86 +145,26 @@ synchronized Map toSpanTags() { /** * Mirrors the Node {@code (typeof value === 'object' && value !== null) ? JSON.stringify(value) : - * String(value)} rule: structured values (objects, arrays) are JSON-stringified; scalars use - * their string form; {@code null} becomes the bare {@code null}. + * String(value)} rule: structured values (Map/List/array) are JSON-stringified; scalars use their + * string form; {@code null} becomes the bare {@code null}. * - *

    On the real OpenFeature object-evaluation path the runtime-default arrives wrapped in a - * {@link Value}; we unwrap it to its native representation first so a structured default - * serializes to JSON (matching Node's {@code JSON.stringify} of the equivalent JS object) instead - * of {@code Value.toString()} (which is {@code "Value(innerObject=...)"}). The scalar cases - * ({@code String}/{@code Boolean}/number) collapse to the same string form Node produces. + *

    The value has already been unwrapped to a native Java type by the capture side (any + * OpenFeature {@code Value} is converted to Map/List/scalar before the seam), so no OpenFeature + * type ever reaches here. */ static String stringifyDefault(final Object value) { - final Object unwrapped = value instanceof Value ? unwrapValue((Value) value) : value; - if (unwrapped == null) { + if (value == null) { return "null"; } - if (unwrapped instanceof Map - || unwrapped instanceof Iterable - || unwrapped.getClass().isArray()) { - return toJsonValue(unwrapped); - } - if (unwrapped instanceof CharSequence || unwrapped instanceof Character) { - return unwrapped.toString(); - } - // Numbers / booleans / Instant — their string form matches what Node's String(value) emits for - // these scalar cases. - return String.valueOf(unwrapped); - } - - /** - * Recursively unwraps an OpenFeature {@link Value} into its native Java representation: - * structures become {@code Map}, lists become {@code List}, and scalars - * become their boxed value (or {@code null}). Nested {@link Value}s are unwrapped at every level - * so a structure containing further structures/lists serializes correctly. - */ - private static Object unwrapValue(final Value value) { - if (value == null || value.isNull()) { - return null; - } - if (value.isStructure()) { - final Structure structure = value.asStructure(); - final Map map = new LinkedHashMap<>(); - if (structure != null) { - for (final String key : structure.keySet()) { - map.put(key, unwrapValue(structure.getValue(key))); - } - } - return map; - } - if (value.isList()) { - final java.util.List list = value.asList(); - final java.util.List out = new java.util.ArrayList<>(list == null ? 0 : list.size()); - if (list != null) { - for (final Value element : list) { - out.add(unwrapValue(element)); - } - } - return out; - } - if (value.isBoolean()) { - return value.asBoolean(); - } - if (value.isString()) { - return value.asString(); - } - if (value.isNumber()) { - // Preserve integral vs fractional so the rendered JSON number matches Node. - final Double d = value.asDouble(); - if (d != null && d == Math.rint(d) && !Double.isInfinite(d)) { - final Integer i = value.asInteger(); - if (i != null) { - return i; - } - } - return d; + if (value instanceof Map || value instanceof Iterable || value.getClass().isArray()) { + return toJsonValue(value); } - final Instant instant = value.asInstant(); - if (instant != null) { - return instant.toString(); + if (value instanceof CharSequence || value instanceof Character) { + return value.toString(); } - // Unknown shape: fall back to the wrapped object's own representation. - return value.asObject(); + // Numbers / booleans — their string form matches what Node's String(value) emits for these + // scalar cases. + return String.valueOf(value); } /** UTF-8-safe truncation: never split a surrogate pair at the {@code maxChars} boundary. */ @@ -262,8 +205,8 @@ private static String toJsonValue(final Object value) { @SuppressWarnings("unchecked") private static void appendJsonValue(final StringBuilder sb, final Object value) { - // Callers pass values already unwrapped to native form by stringifyDefault/unwrapValue, so no - // OpenFeature Value ever reaches here. + // Callers pass values already unwrapped to native form by the capture side, so no OpenFeature + // Value ever reaches here. if (value == null) { sb.append("null"); } else if (value instanceof Map) { diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentInterceptor.java similarity index 53% rename from products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java rename to products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentInterceptor.java index 988c04457c9..7f3cf0d5ff0 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentInterceptor.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentInterceptor.java @@ -1,32 +1,24 @@ -package datadog.trace.api.openfeature; +package com.datadog.featureflag; import datadog.trace.api.interceptor.MutableSpan; import datadog.trace.api.interceptor.TraceInterceptor; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import java.util.Collection; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; /** - * Process-wide {@link TraceInterceptor} that writes the accumulated {@code ffe_*} tags onto the - * local root span when a trace actually completes. This is the WRITE half of the - * capture-vs-write split — the CAPTURE half is {@link SpanEnrichmentHook}, which fills the active - * {@link SpanEnrichmentStates} store during flag evaluation. + * {@link TraceInterceptor} that writes the accumulated {@code ffe_*} tags onto the local root span + * when a trace actually completes. This is the WRITE half of the capture-vs-write split — + * the CAPTURE half is the agent-side {@link SpanEnrichmentWriter}, which fills the {@link + * SpanEnrichmentStates} store from flag-evaluation seam events. * - *

    Reconfiguration safety

    + *

    Single agent-side owner

    * - *

    The tracer holds interceptors in an add-only, priority-keyed set with no public removal API: a - * given priority can be occupied by exactly one interceptor for the life of the tracer. If each - * provider registered its own interceptor at the same priority, the first registration would win - * forever; after that provider closed, every later gate-on provider would be rejected as a - * duplicate and enrichment would be permanently disabled. - * - *

    To survive provider close/reopen we therefore register a single, long-lived delegating - * interceptor ({@link #INSTANCE}) exactly once, and {@linkplain #bind(SpanEnrichmentStates) rebind} - * it to whichever provider is currently active. A provider {@linkplain - * #unbind(SpanEnrichmentStates) unbinds} on close. When no provider is bound the interceptor is - * inert; a fresh provider rebinds it and enrichment resumes — no second registration is ever - * attempted. + *

    Exactly one interceptor is created and owned by the {@link SpanEnrichmentWriter} (agent + * classloader) and registered with the tracer at most once, lazily on the first enrichment event. + * Because the owner lives in the stable agent classloader — not per-provider in an application + * classloader — there is no rebinding, no reconfiguration hazard, and no application-classloader + * pinning. * *

    Partial-flush correctness

    * @@ -45,12 +37,8 @@ * the accumulator and write tags onto a not-yet-finished root, silently dropping all pre-flush * enrichment — exactly the long-running-trace data-loss bug. * - *

    State ownership

    - * - *

    State lives in the per-provider {@link SpanEnrichmentStates} store that is currently bound. - * {@link #unbind(SpanEnrichmentStates)} clears only the store it unbinds, so one provider's close - * can never wipe another's in-flight state. The store is weak-keyed by the local-root span, so a - * trace that never reaches this interceptor is collected with its root and cannot leak unboundedly. + *

    The store is weak-keyed by the local-root span, so a trace that never reaches this interceptor + * is collected with its root and cannot leak unboundedly. * *

    All work is wrapped in try/catch — enrichment must NEVER break trace finish. */ @@ -63,71 +51,24 @@ final class SpanEnrichmentInterceptor implements TraceInterceptor { */ static final int PRIORITY = 4; - /** The single, long-lived interceptor registered with the tracer (reconfiguration safety). */ - static final SpanEnrichmentInterceptor INSTANCE = new SpanEnrichmentInterceptor(); - - // Whether INSTANCE has been accepted by a real tracer. Registration can legitimately fail when - // the global tracer is still the no-op (not yet installed); in that case we leave this false so a - // later provider retries. Once true we never re-attempt (the interceptor is in for good). - private final AtomicBoolean registered = new AtomicBoolean(false); - - // The store of the currently-active provider. null when no gate-on provider is bound, in which - // case the interceptor is inert. Volatile: the eval/bind threads write, the trace-write thread - // reads. - private volatile SpanEnrichmentStates activeStates; - - private SpanEnrichmentInterceptor() {} - - /** - * Idempotently registers {@link #INSTANCE} with the tracer via {@code registrar}. Safe to call - * from every gate-on provider: the first successful registration wins and subsequent calls are - * no-ops. If registration fails because the tracer is not yet installed, a later call retries. - */ - static void ensureRegistered(final Provider.TraceInterceptorRegistrar registrar) { - if (INSTANCE.registered.get()) { - return; - } - if (registrar.register(INSTANCE)) { - INSTANCE.registered.set(true); - } - } - - /** Binds the active store to {@code states}, displacing any previously-bound provider. */ - void bind(final SpanEnrichmentStates states) { - this.activeStates = states; - } - - /** - * Unbinds {@code states} if it is still the active store and clears it (cleanup on provider - * close). If a newer provider has already rebound the interceptor, this is a no-op so the new - * provider's in-flight state is left untouched. - */ - void unbind(final SpanEnrichmentStates states) { - if (this.activeStates == states) { - this.activeStates = null; - } - if (states != null) { - states.clear(); - } - } + private final SpanEnrichmentStates states; - /** The currently-bound store, or {@code null} when the interceptor is inert. */ - SpanEnrichmentStates activeStates() { - return activeStates; + SpanEnrichmentInterceptor(final SpanEnrichmentStates states) { + this.states = states; } @Override public Collection onTraceComplete( final Collection trace) { try { - final SpanEnrichmentStates states = this.activeStates; - if (states == null || trace == null || trace.isEmpty()) { + // Fast path: no accumulated state at all → skip the per-flush scan + lock entirely. This is + // the common case for services that never evaluate a flag on a given trace. + if (trace == null || trace.isEmpty() || states.isEmpty()) { return trace; } // Resolve the local root for this fragment, then require that the root is actually PRESENT in // this collection. A partial flush excludes the still-open root, so its absence means "not - // the - // final write" — keep the accumulator and bail. + // the final write" — keep the accumulator and bail. final MutableSpan localRoot = findLocalRootInFragment(trace); if (!(localRoot instanceof AgentSpan)) { return trace; // partial flush, or no resolvable in-fragment root: keep state untouched diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentStates.java similarity index 81% rename from products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java rename to products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentStates.java index 50a7ab60db6..7231466cd1b 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentStates.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentStates.java @@ -1,12 +1,11 @@ -package datadog.trace.api.openfeature; +package com.datadog.featureflag; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import java.util.Map; import java.util.WeakHashMap; /** - * Instance-owned store of per-trace {@link SpanEnrichmentAccumulator} state, keyed by the - * local-root span object. + * Store of per-trace {@link SpanEnrichmentAccumulator} state, keyed by the local-root span object. * *

    Weak keys. The map is a {@link WeakHashMap} keyed by the local-root {@link AgentSpan} * instance. The accumulator is reachable only while its local-root span is, so a trace that never @@ -19,11 +18,11 @@ * trace-id hex string) is inherently unique per trace: two distinct traces have distinct root * objects and can never share an accumulator. * - *

    Each {@link SpanEnrichmentHook}/{@link SpanEnrichmentInterceptor} pair shares one store - * instance. Because the store is instance-owned rather than a shared static, one provider's cleanup - * clears only its own state and can never wipe another (still-active) provider's in-flight entries. + *

    A single store is owned by the agent-side {@link SpanEnrichmentWriter} and shared with its + * {@link SpanEnrichmentInterceptor}. The writer accumulates (from the flag-eval seam); the + * interceptor reads + removes (trace-write thread). * - *

    Thread-safety: all access is guarded by the intrinsic lock on this instance. The hook writes + *

    Thread-safety: all access is guarded by the intrinsic lock on this instance. The writer writes * (eval thread) and the interceptor reads+removes (trace-write thread) concurrently, so every * mutator and accessor synchronizes. Contention is low — operations are O(1) map touches. */ @@ -49,7 +48,7 @@ synchronized SpanEnrichmentAccumulator remove(final AgentSpan root) { return states.remove(root); } - /** Clears all tracked state (cleanup on provider close / unbind). */ + /** Clears all tracked state (cleanup on shutdown). */ synchronized void clear() { states.clear(); } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java new file mode 100644 index 00000000000..0430c8f34b2 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java @@ -0,0 +1,121 @@ +package com.datadog.featureflag; + +import datadog.trace.api.GlobalTracer; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.SpanEnrichmentEvent; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Agent-side owner of APM feature-flag span enrichment. This is the WRITE tier of the + * capture-vs-write split: it listens on {@link FeatureFlaggingGateway} for {@link + * SpanEnrichmentEvent}s dispatched by the published {@code dd-openfeature} provider during flag + * evaluation, resolves the active local-root span, and accumulates per-trace state that a {@link + * SpanEnrichmentInterceptor} later flushes onto the root when the trace completes. + * + *

    Living in the agent classloader (rather than the application-loaded provider) means the tracer + * API ({@link AgentSpan}/{@link AgentTracer}/{@link GlobalTracer}/{@code TraceInterceptor}) is used + * only here — the published product API never depends on {@code internal-api}. It also gives span + * enrichment a single, stable owner: no per-provider rebinding, no reconfiguration hazard, and no + * application-classloader pinning. + * + *

    Zero idle overhead when off. When the span-enrichment gate is off the provider adds no + * capture hook, so no seam events are dispatched, this listener never runs, and the interceptor is + * never registered — the tracer's write path is untouched. The interceptor is registered lazily on + * the first enrichment event, so a service that enables the feature but never evaluates a flag on a + * traced request still pays nothing. + * + *

    All work is wrapped in try/catch — enrichment must NEVER break flag evaluation. + */ +public final class SpanEnrichmentWriter implements FeatureFlaggingGateway.SpanEnrichmentListener { + + /** + * Resolves the local-root span for the active trace. Injectable so tests need no static mocks. + */ + interface RootSpanResolver { + AgentSpan activeLocalRoot(); + } + + private static final RootSpanResolver DEFAULT_RESOLVER = + () -> { + final AgentSpan active = AgentTracer.activeSpan(); + if (active == null) { + return null; + } + final AgentSpan localRoot = active.getLocalRootSpan(); + return localRoot != null ? localRoot : active; + }; + + private final SpanEnrichmentStates states = new SpanEnrichmentStates(); + private final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); + private final RootSpanResolver rootSpanResolver; + // Registered with the tracer at most once, lazily on the first enrichment event. + private final AtomicBoolean interceptorRegistered = new AtomicBoolean(false); + + public SpanEnrichmentWriter() { + this(DEFAULT_RESOLVER); + } + + SpanEnrichmentWriter(final RootSpanResolver rootSpanResolver) { + this.rootSpanResolver = rootSpanResolver; + } + + /** Starts listening for enrichment events. */ + public void init() { + FeatureFlaggingGateway.addSpanEnrichmentListener(this); + } + + /** Stops listening and drops any residual state. */ + public void close() { + FeatureFlaggingGateway.removeSpanEnrichmentListener(this); + states.clear(); + } + + @Override + public void accept(final SpanEnrichmentEvent event) { + if (event == null) { + return; + } + try { + ensureInterceptorRegistered(); + final AgentSpan root = rootSpanResolver.activeLocalRoot(); + if (root == null) { + return; // no active span → nothing to enrich + } + final SpanEnrichmentAccumulator state = states.getOrCreate(root); + if (event.hasSerialId()) { + final int serialId = event.serialId(); + state.addSerialId(serialId); + if (event.doLog() && event.targetingKey() != null) { + state.addSubject(event.targetingKey(), serialId); + } + } else if (event.flagKey() != null) { + state.addDefault(event.flagKey(), event.defaultValue()); + } + } catch (final Throwable t) { + // Never let span enrichment break flag evaluation. + } + } + + private void ensureInterceptorRegistered() { + if (interceptorRegistered.compareAndSet(false, true)) { + try { + GlobalTracer.get().addTraceInterceptor(interceptor); + } catch (final Throwable t) { + // Tracer not yet installed (e.g. no-op placeholder): allow a later event to retry. + interceptorRegistered.set(false); + } + } + } + + // ---- test-only accessors ---- + + SpanEnrichmentStates states() { + return states; + } + + SpanEnrichmentInterceptor interceptor() { + return interceptor; + } +} diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ULeb128Encoder.java similarity index 98% rename from products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java rename to products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ULeb128Encoder.java index 14e441a28b4..2475be335b0 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ULeb128Encoder.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ULeb128Encoder.java @@ -1,4 +1,4 @@ -package datadog.trace.api.openfeature; +package com.datadog.featureflag; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentAccumulatorTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentAccumulatorTest.java new file mode 100644 index 00000000000..06b5cec00ab --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentAccumulatorTest.java @@ -0,0 +1,159 @@ +package com.datadog.featureflag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Base64; +import java.util.Collections; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; +import org.junit.jupiter.api.Test; + +/** + * Write-tier codec + accumulator suite. The encoding, limits, and tag shapes are FROZEN against the + * Node reference ({@code dd-trace-js#8343}); the golden vector {@code {100,108,128,130} -> + * "ZAgUAg=="} anchors byte-for-byte cross-SDK parity. + */ +class SpanEnrichmentAccumulatorTest { + + // Test-only decode oracle for the ULEB128 delta-varint codec — the production code only encodes. + private static SortedSet decodeDeltaVarint(final String encoded) { + final SortedSet result = new TreeSet<>(); + if (encoded == null || encoded.isEmpty()) { + return result; + } + final byte[] bytes = Base64.getDecoder().decode(encoded); + int previous = 0; + int index = 0; + while (index < bytes.length) { + long value = 0; + int shift = 0; + while (true) { + final byte b = bytes[index++]; + value |= ((long) (b & 0x7F)) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + previous += (int) value; + result.add(previous); + } + return result; + } + + @Test + void codecGoldenVectorAndRoundTrip() { + final SortedSet ids = new TreeSet<>(); + ids.add(100); + ids.add(108); + ids.add(128); + ids.add(130); + final String encoded = ULeb128Encoder.encodeDeltaVarint(ids); + assertEquals("ZAgUAg==", encoded, "golden vector must match the frozen Node contract"); + assertEquals(ids, decodeDeltaVarint(encoded)); + assertEquals("", ULeb128Encoder.encodeDeltaVarint(Collections.emptySet())); + final SortedSet withDup = new TreeSet<>(ids); + withDup.add(100); + assertEquals( + encoded, ULeb128Encoder.encodeDeltaVarint(withDup), "duplicates do not change bytes"); + } + + @Test + void flagsEncTagFromAccumulatedSerialIds() { + final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); + acc.addSerialId(100); + acc.addSerialId(108); + acc.addSerialId(128); + acc.addSerialId(130); + acc.addSerialId(100); // dedupe + assertTrue(acc.hasData()); + assertEquals("ZAgUAg==", acc.toSpanTags().get(SpanEnrichmentAccumulator.TAG_FLAGS_ENC)); + } + + @Test + void max200SerialIdsEnforced() { + final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); + for (int i = 0; i < 300; i++) { + acc.addSerialId(i); + } + assertEquals( + SpanEnrichmentAccumulator.MAX_SERIAL_IDS, + acc.serialIdsView().size(), + "serial ids must be capped at 200"); + } + + @Test + void subjectAndPerSubjectExperimentCaps() { + // per-subject experiment cap: 20 max + final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); + for (int i = 0; i < 25; i++) { + acc.addSubject("subjectX", i); + } + final SortedSet decoded = + decodeDeltaVarint( + acc.toSpanTags() + .get(SpanEnrichmentAccumulator.TAG_SUBJECTS_ENC) + .replaceAll("^\\{\"[a-f0-9]+\":\"", "") + .replaceAll("\"\\}$", "")); + assertEquals(SpanEnrichmentAccumulator.MAX_EXPERIMENTS_PER_SUBJECT, decoded.size()); + + // subject cap: 10 max distinct subjects + final SpanEnrichmentAccumulator acc2 = new SpanEnrichmentAccumulator(); + for (int i = 0; i < 15; i++) { + acc2.addSubject("subject-" + i, i); + } + assertEquals(SpanEnrichmentAccumulator.MAX_SUBJECTS, acc2.subjectCount()); + } + + @Test + void runtimeDefaultJsonStringifyAndTruncation() { + // native object -> JSON, not toString + assertEquals( + "{\"a\":\"b\"}", + SpanEnrichmentAccumulator.stringifyDefault(Collections.singletonMap("a", "b"))); + // scalar string -> as-is + assertEquals("hello", SpanEnrichmentAccumulator.stringifyDefault("hello")); + // null -> "null" + assertEquals("null", SpanEnrichmentAccumulator.stringifyDefault(null)); + // native list -> JSON array + assertEquals( + "[\"a\",2,true]", + SpanEnrichmentAccumulator.stringifyDefault(java.util.Arrays.asList("a", 2, true))); + + // 64-char truncation + final StringBuilder longValue = new StringBuilder(); + for (int i = 0; i < 100; i++) { + longValue.append('x'); + } + final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); + acc.addDefault("flag", longValue.toString()); + final String tag = acc.toSpanTags().get(SpanEnrichmentAccumulator.TAG_RUNTIME_DEFAULTS); + final String expectedValue = + longValue.substring(0, SpanEnrichmentAccumulator.MAX_DEFAULT_VALUE_LENGTH); + assertEquals("{\"flag\":\"" + expectedValue + "\"}", tag); + + // first-wins + acc.addDefault("flag", "second"); + assertEquals(1, acc.defaultCount()); + } + + @Test + void maxDefaultsEnforced() { + final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); + for (int i = 0; i < 10; i++) { + acc.addDefault("flag-" + i, "v"); + } + assertEquals(SpanEnrichmentAccumulator.MAX_DEFAULTS, acc.defaultCount()); + } + + @Test + void noDataWhenEmpty() { + final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); + assertFalse(acc.hasData()); + final Map tags = acc.toSpanTags(); + assertTrue(tags.isEmpty()); + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentInterceptorTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentInterceptorTest.java new file mode 100644 index 00000000000..a82d7e301af --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentInterceptorTest.java @@ -0,0 +1,162 @@ +package com.datadog.featureflag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.trace.api.interceptor.MutableSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * Write-tier interceptor suite: partial-flush correctness (the long-running-trace regression), + * identity keying, and the no-state fast path. dd-trace-core runs {@code onTraceComplete} on every + * flush; a partial flush excludes the still-open root, so the interceptor must only flush+remove + * when the local root is present in the fragment. + */ +class SpanEnrichmentInterceptorTest { + + /** A mock local-root span reporting itself as its own local root. */ + private static AgentSpan rootSpan() { + final AgentSpan root = mock(AgentSpan.class); + when(root.getLocalRootSpan()).thenReturn(root); + return root; + } + + /** A mock child span whose local root is {@code root} but which is itself NOT the root. */ + private static AgentSpan childOf(final AgentSpan root) { + final AgentSpan child = mock(AgentSpan.class); + when(child.getLocalRootSpan()).thenReturn(root); + return child; + } + + @Test + void priorityIsUnique() { + assertEquals(4, new SpanEnrichmentInterceptor(new SpanEnrichmentStates()).priority()); + } + + @Test + void emptyOrNoStateIsANoOpFastPath() { + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); + // empty trace + assertTrue(interceptor.onTraceComplete(Collections.emptyList()).isEmpty()); + // no accumulated state => fast path, never touches the span + final AgentSpan root = rootSpan(); + interceptor.onTraceComplete(Collections.singletonList(root)); + verify(root, never()).setTag(anyString(), anyString()); + } + + @Test + void finalFlushWithRootPresentWritesTags() { + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); + final AgentSpan root = rootSpan(); + states.getOrCreate(root).addSerialId(5); + + interceptor.onTraceComplete(Collections.singletonList(root)); + + verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "BQ=="); // {5} -> 0x05 + assertTrue(states.isEmpty(), "state must be removed on the final flush"); + } + + @Test + void partialFlushExcludingRootPreservesState() { + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); + final AgentSpan root = rootSpan(); + states.getOrCreate(root).addSerialId(100); + states.getOrCreate(root).addSerialId(108); + + // PARTIAL flush: a fragment of children only — the open root is NOT in the collection. + final AgentSpan child1 = childOf(root); + final AgentSpan child2 = childOf(root); + interceptor.onTraceComplete(Arrays.asList(child1, child2)); + + verify(child1, never()).setTag(anyString(), anyString()); + verify(child2, never()).setTag(anyString(), anyString()); + verify(root, never()).setTag(anyString(), anyString()); + final SpanEnrichmentAccumulator surviving = states.peek(root); + assertNotNull(surviving, "partial flush must NOT remove the accumulator"); + assertTrue(surviving.serialIdsView().contains(100) && surviving.serialIdsView().contains(108)); + + // more evaluations, then the FINAL flush with the root present + states.getOrCreate(root).addSerialId(128); + states.getOrCreate(root).addSerialId(130); + final AgentSpan lateChild = childOf(root); + interceptor.onTraceComplete(Arrays.asList(lateChild, root)); + + verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZAgUAg=="); // {100,108,128,130} + assertTrue(states.isEmpty(), "state removed only on the final flush"); + } + + @Test + void childOnlyFragmentNeverTagsAChildSpan() { + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); + final AgentSpan root = rootSpan(); + states.getOrCreate(root).addSerialId(7); + + final AgentSpan child = childOf(root); + interceptor.onTraceComplete(Collections.singletonList(child)); + + verify(child, never()).setTag(anyString(), anyString()); + verify(root, never()).setTag(anyString(), anyString()); + assertNotNull(states.peek(root), "child-only fragment must keep state"); + } + + @Test + void distinctTracesDoNotMerge() { + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); + final AgentSpan rootA = rootSpan(); + final AgentSpan rootB = rootSpan(); + states.getOrCreate(rootA).addSerialId(100); + states.getOrCreate(rootB).addSerialId(130); + assertEquals(2, states.size(), "distinct root spans must not share an accumulator"); + + interceptor.onTraceComplete(Collections.singletonList(rootA)); + verify(rootA).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ZA=="); // {100} -> 0x64 + interceptor.onTraceComplete(Collections.singletonList(rootB)); + verify(rootB).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "ggE="); // {130} -> 0x82 0x01 + } + + @Test + void neverCompletingTracesAreNotCapped() { + // No cap / eviction: while their local-root spans are reachable, every never-completing trace + // keeps its accumulator. Weak-key reclamation (not a fixed count) bounds memory. + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + final int churn = 20_000; // well beyond the old 4096 cap + final List liveRoots = new ArrayList<>(churn); + for (int i = 0; i < churn; i++) { + final AgentSpan root = rootSpan(); + liveRoots.add(root); + states.getOrCreate(root).addSerialId(i % 1000 + 1); + } + assertEquals(churn, states.size(), "no cap/eviction: every live root keeps its accumulator"); + } + + @Test + void keyingSymmetryRootResolvedFromFragmentMatchesCapturedRoot() { + final SpanEnrichmentStates states = new SpanEnrichmentStates(); + final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); + final AgentSpan root = rootSpan(); + states.getOrCreate(root).addSerialId(5); + + // The interceptor resolves the root from the fragment (root + a late child); it must be the + // same object so the remove hits the captured accumulator. + final AgentSpan child = childOf(root); + interceptor.onTraceComplete(Arrays.asList(child, root)); + verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "BQ=="); // {5} -> 0x05 + assertTrue(states.isEmpty()); + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentWriterTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentWriterTest.java new file mode 100644 index 00000000000..1f19ed187b3 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentWriterTest.java @@ -0,0 +1,114 @@ +package com.datadog.featureflag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.trace.api.featureflag.SpanEnrichmentEvent; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import java.util.Collections; +import org.junit.jupiter.api.Test; + +/** + * Write-tier listener suite: the agent-side {@link SpanEnrichmentWriter} translates {@link + * SpanEnrichmentEvent}s from the flag-eval seam into per-local-root accumulator state (resolving + * the active root through an injectable resolver so no static tracer is needed). + */ +class SpanEnrichmentWriterTest { + + private static AgentSpan rootSpan() { + final AgentSpan root = mock(AgentSpan.class); + when(root.getLocalRootSpan()).thenReturn(root); + return root; + } + + private static SpanEnrichmentWriter writerFor(final AgentSpan root) { + return new SpanEnrichmentWriter(() -> root); + } + + @Test + void serialIdWithDoLogAndTargetingKeyRecordsSerialAndSubject() { + final AgentSpan root = rootSpan(); + final SpanEnrichmentWriter writer = writerFor(root); + writer.accept(SpanEnrichmentEvent.serialId(42, true, "user-1")); + + final SpanEnrichmentAccumulator state = writer.states().peek(root); + assertNotNull(state); + assertTrue(state.serialIdsView().contains(42)); + assertEquals(1, state.subjectCount(), "doLog=true + targeting key => subject recorded"); + } + + @Test + void serialIdWithoutDoLogRecordsNoSubject() { + final AgentSpan root = rootSpan(); + final SpanEnrichmentWriter writer = writerFor(root); + writer.accept(SpanEnrichmentEvent.serialId(7, false, "user-1")); + + final SpanEnrichmentAccumulator state = writer.states().peek(root); + assertTrue(state.serialIdsView().contains(7)); + assertEquals(0, state.subjectCount(), "doLog=false must not record a subject"); + } + + @Test + void serialIdWithNullTargetingKeyRecordsNoSubject() { + final AgentSpan root = rootSpan(); + final SpanEnrichmentWriter writer = writerFor(root); + writer.accept(SpanEnrichmentEvent.serialId(9, true, null)); + + final SpanEnrichmentAccumulator state = writer.states().peek(root); + assertTrue(state.serialIdsView().contains(9)); + assertEquals(0, state.subjectCount(), "no targeting key => no subject"); + } + + @Test + void runtimeDefaultRecordsDefault() { + final AgentSpan root = rootSpan(); + final SpanEnrichmentWriter writer = writerFor(root); + writer.accept(SpanEnrichmentEvent.runtimeDefault("flag", Collections.singletonMap("k", "v"))); + + final SpanEnrichmentAccumulator state = writer.states().peek(root); + assertNotNull(state); + assertEquals(1, state.defaultCount()); + } + + @Test + void noActiveSpanAccumulatesNothing() { + final SpanEnrichmentWriter writer = new SpanEnrichmentWriter(() -> null); + writer.accept(SpanEnrichmentEvent.serialId(1, true, "user-1")); + assertTrue(writer.states().isEmpty(), "no active span => no accumulator state"); + } + + @Test + void nullEventIsANoOp() { + final AgentSpan root = rootSpan(); + final SpanEnrichmentWriter writer = writerFor(root); + writer.accept(null); + assertTrue(writer.states().isEmpty()); + } + + @Test + void endToEndAccumulateThenFlush() { + final AgentSpan root = rootSpan(); + final SpanEnrichmentWriter writer = writerFor(root); + writer.accept(SpanEnrichmentEvent.serialId(5, false, null)); + + writer.interceptor().onTraceComplete(Collections.singletonList(root)); + verify(root).setTag(SpanEnrichmentAccumulator.TAG_FLAGS_ENC, "BQ=="); // {5} -> 0x05 + assertTrue(writer.states().isEmpty(), "flush removes the accumulated state"); + } + + @Test + void distinctEventsUnderSameRootShareAccumulator() { + final AgentSpan root = rootSpan(); + final SpanEnrichmentWriter writer = writerFor(root); + writer.accept(SpanEnrichmentEvent.serialId(100, false, null)); + writer.accept(SpanEnrichmentEvent.serialId(108, false, null)); + assertEquals(1, writer.states().size(), "same root => one shared accumulator"); + verify(root, never()).setTag(anyString(), anyString()); + } +} From ac8195b39cf8b5f0b271acac2c608b26836099ec Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 8 Jul 2026 12:00:32 +0300 Subject: [PATCH 12/18] Use JsonWriter instead of hand-written solution --- .../feature-flagging-lib/build.gradle.kts | 3 + .../SpanEnrichmentAccumulator.java | 118 ++++++------------ .../SpanEnrichmentAccumulatorTest.java | 18 +++ 3 files changed, 61 insertions(+), 78 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 584e867f321..2008af2b675 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -26,6 +26,9 @@ dependencies { // an agent-side module (shaded into the agent, never published), so depending on the tracer's // internal API is expected — unlike the published dd-openfeature product API. compileOnly(project(":internal-api")) + // Platform JSON writer for the ffe_* tag values (provided by the agent at runtime, like the + // tracer API above; also reachable transitively via :internal-api). + compileOnly(project(":components:json")) testImplementation(libs.bundles.junit5) testImplementation(libs.bundles.mockito) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentAccumulator.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentAccumulator.java index 1b90b3d265f..a14274432b0 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentAccumulator.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentAccumulator.java @@ -1,5 +1,6 @@ package com.datadog.featureflag; +import datadog.json.JsonWriter; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; @@ -179,104 +180,65 @@ static String utf8SafeTruncate(final String value, final int maxChars) { return value.substring(0, end); } - /** Serializes a String->String map to a compact JSON object string (keys + values escaped). */ + /** + * Serializes a String->String map to a compact JSON object string using the platform {@link + * JsonWriter}. + * + *

    Consumers of these tags parse them as JSON (the backend enricher via Jackson, the parametric + * system-tests via {@code json.loads}), so the writer's escaping (e.g. {@code /} → {@code \/}, + * non-ASCII → {@code \\uXXXX}) is round-trip-equivalent and byte-parity with the JS reference is + * not required. + */ static String toJsonObject(final Map map) { - final StringBuilder sb = new StringBuilder(); - sb.append('{'); - boolean first = true; - for (final Map.Entry entry : map.entrySet()) { - if (!first) { - sb.append(','); + try (JsonWriter writer = new JsonWriter()) { + writer.beginObject(); + for (final Map.Entry entry : map.entrySet()) { + writer.name(entry.getKey()).value(entry.getValue()); } - first = false; - appendJsonString(sb, entry.getKey()); - sb.append(':'); - appendJsonString(sb, entry.getValue()); + writer.endObject(); + return writer.toString(); } - sb.append('}'); - return sb.toString(); } private static String toJsonValue(final Object value) { - final StringBuilder sb = new StringBuilder(); - appendJsonValue(sb, value); - return sb.toString(); + try (JsonWriter writer = new JsonWriter()) { + writeJsonValue(writer, value); + return writer.toString(); + } } @SuppressWarnings("unchecked") - private static void appendJsonValue(final StringBuilder sb, final Object value) { + private static void writeJsonValue(final JsonWriter writer, final Object value) { // Callers pass values already unwrapped to native form by the capture side, so no OpenFeature // Value ever reaches here. if (value == null) { - sb.append("null"); + writer.nullValue(); } else if (value instanceof Map) { - sb.append('{'); - boolean first = true; + writer.beginObject(); for (final Map.Entry entry : ((Map) value).entrySet()) { - if (!first) { - sb.append(','); - } - first = false; - appendJsonString(sb, String.valueOf(entry.getKey())); - sb.append(':'); - appendJsonValue(sb, entry.getValue()); + writer.name(String.valueOf(entry.getKey())); + writeJsonValue(writer, entry.getValue()); } - sb.append('}'); + writer.endObject(); } else if (value instanceof Iterable) { - sb.append('['); - boolean first = true; + writer.beginArray(); for (final Object element : (Iterable) value) { - if (!first) { - sb.append(','); - } - first = false; - appendJsonValue(sb, element); + writeJsonValue(writer, element); } - sb.append(']'); - } else if (value instanceof CharSequence || value instanceof Character) { - appendJsonString(sb, value.toString()); - } else if (value instanceof Number || value instanceof Boolean) { - sb.append(String.valueOf(value)); + writer.endArray(); + } else if (value instanceof Boolean) { + writer.value((Boolean) value); + } else if (value instanceof Integer + || value instanceof Long + || value instanceof Short + || value instanceof Byte) { + writer.value(((Number) value).longValue()); + } else if (value instanceof Number) { + writer.value(((Number) value).doubleValue()); } else { - appendJsonString(sb, String.valueOf(value)); - } - } - - private static void appendJsonString(final StringBuilder sb, final String value) { - sb.append('"'); - for (int i = 0; i < value.length(); i++) { - final char c = value.charAt(i); - switch (c) { - case '"': - sb.append("\\\""); - break; - case '\\': - sb.append("\\\\"); - break; - case '\n': - sb.append("\\n"); - break; - case '\r': - sb.append("\\r"); - break; - case '\t': - sb.append("\\t"); - break; - case '\b': - sb.append("\\b"); - break; - case '\f': - sb.append("\\f"); - break; - default: - if (c < 0x20) { - sb.append(String.format("\\u%04x", (int) c)); - } else { - sb.append(c); - } - } + // CharSequence / Character / anything else → string form. + writer.value(value.toString()); } - sb.append('"'); } // ---- test-only accessors ---- diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentAccumulatorTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentAccumulatorTest.java index 06b5cec00ab..c391d5b1027 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentAccumulatorTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentAccumulatorTest.java @@ -140,6 +140,24 @@ void runtimeDefaultJsonStringifyAndTruncation() { assertEquals(1, acc.defaultCount()); } + @Test + void jsonUsesPlatformWriterEscaping() { + // The platform JsonWriter escapes '/' -> \/ and non-ASCII -> \uXXXX. That differs from the JS + // reference bytes but is round-trip-equivalent: all consumers JSON-parse these tags (backend + // Jackson, system-tests json.loads), so byte-parity is not required. '/' matters in practice + // because ffe_subjects_enc values are base64 (which can contain '/'). + assertEquals( + "{\"h\":\"a\\/b\"}", + SpanEnrichmentAccumulator.toJsonObject(Collections.singletonMap("h", "a/b"))); + assertEquals( + "{\"k\":\"caf\\u00E9\"}", + SpanEnrichmentAccumulator.toJsonObject(Collections.singletonMap("k", "café"))); + // nested structured runtime default goes through the same writer + assertEquals( + "{\"a\":\"x\\/y\"}", + SpanEnrichmentAccumulator.stringifyDefault(Collections.singletonMap("a", "x/y"))); + } + @Test void maxDefaultsEnforced() { final SpanEnrichmentAccumulator acc = new SpanEnrichmentAccumulator(); From 77a8ec23ef28d713f6262af939a05b2738eec51e Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 8 Jul 2026 12:03:28 +0300 Subject: [PATCH 13/18] Fix unicode char --- .../featureflag/SpanEnrichmentAccumulatorTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentAccumulatorTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentAccumulatorTest.java index c391d5b1027..8b1ff1cee98 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentAccumulatorTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentAccumulatorTest.java @@ -142,10 +142,11 @@ void runtimeDefaultJsonStringifyAndTruncation() { @Test void jsonUsesPlatformWriterEscaping() { - // The platform JsonWriter escapes '/' -> \/ and non-ASCII -> \uXXXX. That differs from the JS - // reference bytes but is round-trip-equivalent: all consumers JSON-parse these tags (backend - // Jackson, system-tests json.loads), so byte-parity is not required. '/' matters in practice - // because ffe_subjects_enc values are base64 (which can contain '/'). + // The platform JsonWriter escapes '/' as backslash-slash and non-ASCII as a backslash-u escape. + // That differs from the JS reference bytes but is round-trip-equivalent: all consumers + // JSON-parse these tags (backend Jackson, system-tests json.loads), so byte-parity is not + // required. '/' matters in practice because ffe_subjects_enc values are base64 (may contain + // '/'). assertEquals( "{\"h\":\"a\\/b\"}", SpanEnrichmentAccumulator.toJsonObject(Collections.singletonMap("h", "a/b"))); From 5ac008de3bf4691248e5092a9b49d5c738422219 Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 8 Jul 2026 12:20:07 +0300 Subject: [PATCH 14/18] Address review nits: gate enrichment metadata, name constants, reuse digest - Gate the __dd_split_serial_id / __dd_do_log evaluation metadata on the span-enrichment flag so an enabled provider with enrichment off attaches nothing extra per evaluation (DDEvaluator). - Extract the metadata keys to named constants (METADATA_SPLIT_SERIAL_ID / METADATA_DO_LOG) as the single source of truth; SpanEnrichmentHook references them. - Rename the config constant to EXPERIMENTAL_SPAN_ENRICHMENT_ENABLED to make its experimental status obvious at call sites. - Reuse a ThreadLocal for subject hashing instead of allocating a SHA-256 instance per capture (ULeb128Encoder). --- .../api/config/FeatureFlaggingConfig.java | 2 +- .../trace/api/openfeature/DDEvaluator.java | 37 +++++++++++++++--- .../trace/api/openfeature/Provider.java | 7 ++-- .../api/openfeature/SpanEnrichmentHook.java | 5 ++- .../datadog/featureflag/ULeb128Encoder.java | 38 ++++++++++++------- 5 files changed, 63 insertions(+), 26 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index e5a2edb856b..c915effde13 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -10,6 +10,6 @@ public class FeatureFlaggingConfig { * {@code DD_} prefix normalization rule. This is DISTINCT from {@link #FLAGGING_PROVIDER_ENABLED} * and is OFF by default — enabling the provider does not enable span enrichment. */ - public static final String SPAN_ENRICHMENT_ENABLED = + public static final String EXPERIMENTAL_SPAN_ENRICHMENT_ENABLED = "experimental.flagging.provider.span.enrichment.enabled"; } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 88fa7d4812f..f6936632f7c 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -2,6 +2,7 @@ import static java.util.Arrays.asList; +import datadog.trace.api.config.FeatureFlaggingConfig; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.exposure.ExposureEvent; import datadog.trace.api.featureflag.exposure.Subject; @@ -16,6 +17,7 @@ import datadog.trace.api.featureflag.ufc.v1.Split; import datadog.trace.api.featureflag.ufc.v1.ValueType; import datadog.trace.api.featureflag.ufc.v1.Variant; +import datadog.trace.bootstrap.config.provider.ConfigProvider; import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.ImmutableMetadata; @@ -46,6 +48,16 @@ class DDEvaluator implements Evaluator, FeatureFlaggingGateway.ConfigListener { private static final Set> SUPPORTED_RESOLUTION_TYPES = new HashSet<>(asList(String.class, Boolean.class, Integer.class, Double.class, Value.class)); + // Evaluation-metadata keys consumed by the span-enrichment capture hook (see + // SpanEnrichmentHook). Emitted only when the span-enrichment gate is on. + static final String METADATA_SPLIT_SERIAL_ID = "__dd_split_serial_id"; + static final String METADATA_DO_LOG = "__dd_do_log"; + + // Read once: when off, the __dd_* span-enrichment metadata is not attached to evaluations, so an + // enabled provider pays nothing extra unless span enrichment is also enabled. The gate does not + // change at runtime, and this class is loaded lazily (well after startup) so config is ready. + private static final boolean SPAN_ENRICHMENT_ENABLED = readSpanEnrichmentEnabled(); + private final Runnable configCallback; private final AtomicReference configuration = new AtomicReference<>(); private final CountDownLatch initializationLatch = new CountDownLatch(1); @@ -54,6 +66,15 @@ public DDEvaluator(final Runnable configCallback) { this.configCallback = configCallback; } + private static boolean readSpanEnrichmentEnabled() { + try { + return ConfigProvider.getInstance() + .getBoolean(FeatureFlaggingConfig.EXPERIMENTAL_SPAN_ENRICHMENT_ENABLED, false); + } catch (final Throwable t) { + return false; // never let config reading break evaluator class initialization + } + } + @Override public boolean initialize( final long timeout, final TimeUnit unit, final EvaluationContext context) throws Exception { @@ -392,14 +413,18 @@ private static ProviderEvaluation resolveVariant( .addString("flagKey", flag.key) .addString("variationType", flag.variationType.name()) .addString("allocationKey", allocation.key); - // Surface the UFC split's serial id and the allocation's doLog flag for APM span enrichment. + // Surface the UFC split's serial id and the allocation's doLog flag for APM span enrichment — + // only when span enrichment is on, so a provider without enrichment pays nothing extra. // __dd_split_serial_id is omitted when the split carries no serial id; __dd_do_log is always - // present so the span-enrichment hook can decide whether to record the subject. - if (split.serialId != null) { - metadataBuilder.addString("__dd_split_serial_id", split.serialId.toString()); + // present (when enrichment is on) so the span-enrichment hook can decide whether to record the + // subject. + if (SPAN_ENRICHMENT_ENABLED) { + if (split.serialId != null) { + metadataBuilder.addString(METADATA_SPLIT_SERIAL_ID, split.serialId.toString()); + } + metadataBuilder.addString( + METADATA_DO_LOG, String.valueOf(allocation.doLog != null && allocation.doLog)); } - metadataBuilder.addString( - "__dd_do_log", String.valueOf(allocation.doLog != null && allocation.doLog)); final ProviderEvaluation result = ProviderEvaluation.builder() .value(mappedValue) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index 21df800d703..fe2159551d9 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -34,13 +34,14 @@ public class Provider extends EventProvider implements Metadata { /** * Canonical config key for the span-enrichment gate ({@link - * FeatureFlaggingConfig#SPAN_ENRICHMENT_ENABLED}). Read through {@link ConfigProvider} so the - * full precedence applies — system property, env var ({@code + * FeatureFlaggingConfig#EXPERIMENTAL_SPAN_ENRICHMENT_ENABLED}). Read through {@link + * ConfigProvider} so the full precedence applies — system property, env var ({@code * DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED}), and stable config — exactly like * the sibling provider-enabled gate. Distinct from the provider-enabled gate; OFF by default * (experimental opt-in). */ - static final String SPAN_ENRICHMENT_ENABLED_KEY = FeatureFlaggingConfig.SPAN_ENRICHMENT_ENABLED; + static final String SPAN_ENRICHMENT_ENABLED_KEY = + FeatureFlaggingConfig.EXPERIMENTAL_SPAN_ENRICHMENT_ENABLED; private static final Options DEFAULT_OPTIONS = new Options().initTimeout(30, SECONDS); private volatile Evaluator evaluator; diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java index ef4ce123bc9..b93d249c2b1 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java @@ -41,8 +41,9 @@ */ class SpanEnrichmentHook implements Hook { - static final String METADATA_SERIAL_ID = "__dd_split_serial_id"; - static final String METADATA_DO_LOG = "__dd_do_log"; + // The metadata keys the DDEvaluator attaches for span enrichment (single source of truth there). + static final String METADATA_SERIAL_ID = DDEvaluator.METADATA_SPLIT_SERIAL_ID; + static final String METADATA_DO_LOG = DDEvaluator.METADATA_DO_LOG; @Override public void finallyAfter( diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ULeb128Encoder.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ULeb128Encoder.java index 2475be335b0..7566cf8698d 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ULeb128Encoder.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ULeb128Encoder.java @@ -67,21 +67,31 @@ static String encodeDeltaVarint(final Set serialIds) { * @return the lower-case hex SHA-256 digest */ static String hashTargetingKey(final String value) { - try { - final MessageDigest digest = MessageDigest.getInstance("SHA-256"); - final byte[] hash = digest.digest(value.getBytes(StandardCharsets.UTF_8)); - final StringBuilder hex = new StringBuilder(hash.length * 2); - for (final byte b : hash) { - final int v = b & 0xFF; - if (v < 0x10) { - hex.append('0'); - } - hex.append(Integer.toHexString(v)); + final MessageDigest digest = SHA_256.get(); + digest.reset(); + final byte[] hash = digest.digest(value.getBytes(StandardCharsets.UTF_8)); + final StringBuilder hex = new StringBuilder(hash.length * 2); + for (final byte b : hash) { + final int v = b & 0xFF; + if (v < 0x10) { + hex.append('0'); } - return hex.toString(); - } catch (final NoSuchAlgorithmException e) { - // SHA-256 is mandated by the JLS to be present on every JVM; this is unreachable in practice. - throw new IllegalStateException("SHA-256 algorithm not available", e); + hex.append(Integer.toHexString(v)); } + return hex.toString(); } + + // Per-thread SHA-256 instance: hashing runs on every doLog=true subject capture, so a + // ThreadLocal avoids a provider lookup + allocation per call on that hot path. digest() resets + // the instance after each hash; we also reset() defensively before use. + private static final ThreadLocal SHA_256 = + ThreadLocal.withInitial( + () -> { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (final NoSuchAlgorithmException e) { + // SHA-256 is mandated by the JLS on every JVM; unreachable in practice. + throw new IllegalStateException("SHA-256 algorithm not available", e); + } + }); } From 22255cbb2dd291509a926f176a89d0eb91efbe60 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Thu, 9 Jul 2026 09:48:45 +0200 Subject: [PATCH 15/18] feat(ffe): Introduce config module to decouple feature flagging from tracing API (#11888) --- dd-java-agent/agent-bootstrap/build.gradle | 1 + .../java/datadog/trace/bootstrap/Agent.java | 2 +- dd-trace-api/build.gradle.kts | 1 - .../feature-flagging-api/build.gradle.kts | 6 +- .../trace/api/openfeature/DDEvaluator.java | 2 +- .../trace/api/openfeature/Provider.java | 2 +- .../feature-flagging-config/build.gradle.kts | 12 +++ .../feature-flagging-config/gradle.lockfile | 75 +++++++++++++++++++ .../config/FeatureFlaggingConfig.java | 2 +- settings.gradle.kts | 1 + 10 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-config/build.gradle.kts create mode 100644 products/feature-flagging/feature-flagging-config/gradle.lockfile rename {dd-trace-api/src/main/java/datadog/trace/api => products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag}/config/FeatureFlaggingConfig.java (93%) diff --git a/dd-java-agent/agent-bootstrap/build.gradle b/dd-java-agent/agent-bootstrap/build.gradle index fc5866caebe..c554039afa5 100644 --- a/dd-java-agent/agent-bootstrap/build.gradle +++ b/dd-java-agent/agent-bootstrap/build.gradle @@ -23,6 +23,7 @@ dependencies { api project(':dd-java-agent:agent-debugger:debugger-bootstrap') api project(':components:environment') api project(':components:json') + api project(':products:feature-flagging:feature-flagging-config') api project(':products:metrics:metrics-agent') api libs.instrument.java api libs.slf4j diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 876a07f1fd1..249453c58fb 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -34,7 +34,6 @@ import datadog.trace.api.config.CrashTrackingConfig; import datadog.trace.api.config.CwsConfig; import datadog.trace.api.config.DebuggerConfig; -import datadog.trace.api.config.FeatureFlaggingConfig; import datadog.trace.api.config.GeneralConfig; import datadog.trace.api.config.IastConfig; import datadog.trace.api.config.JmxFetchConfig; @@ -44,6 +43,7 @@ import datadog.trace.api.config.TraceInstrumentationConfig; import datadog.trace.api.config.TracerConfig; import datadog.trace.api.config.UsmConfig; +import datadog.trace.api.featureflag.config.FeatureFlaggingConfig; import datadog.trace.api.gateway.RequestContextSlot; import datadog.trace.api.gateway.SubscriptionService; import datadog.trace.api.git.EmbeddedGitInfoBuilder; diff --git a/dd-trace-api/build.gradle.kts b/dd-trace-api/build.gradle.kts index 9716588d34e..46e975af41f 100644 --- a/dd-trace-api/build.gradle.kts +++ b/dd-trace-api/build.gradle.kts @@ -38,7 +38,6 @@ extra["excludedClassesCoverage"] = listOf( "datadog.trace.api.civisibility.noop.NoOpDDTestSession", "datadog.trace.api.civisibility.noop.NoOpDDTestSuite", "datadog.trace.api.config.AIGuardConfig", - "datadog.trace.api.config.FeatureFlaggingConfig", "datadog.trace.api.config.ProfilingConfig", "datadog.trace.api.interceptor.MutableSpan", "datadog.trace.api.profiling.Profiling", diff --git a/products/feature-flagging/feature-flagging-api/build.gradle.kts b/products/feature-flagging/feature-flagging-api/build.gradle.kts index 2d6308592cb..ba8b43ca32c 100644 --- a/products/feature-flagging/feature-flagging-api/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-api/build.gradle.kts @@ -44,16 +44,12 @@ dependencies { api("dev.openfeature:sdk:1.20.1") compileOnly(project(":products:feature-flagging:feature-flagging-bootstrap")) + compileOnly(project(":products:feature-flagging:feature-flagging-config")) compileOnly(project(":utils:config-utils")) - // Public config constants (FeatureFlaggingConfig). dd-trace-api is the customer-facing public API, - // so this is a valid product-API dependency (previously reached transitively via :internal-api). - // compileOnly: the constant is inlined, and the agent runtime provides the class. - compileOnly(project(":dd-trace-api")) compileOnly("io.opentelemetry:opentelemetry-api:1.47.0") testImplementation(project(":products:feature-flagging:feature-flagging-bootstrap")) testImplementation(project(":utils:config-utils")) - testImplementation(project(":dd-trace-api")) testImplementation("io.opentelemetry:opentelemetry-api:1.47.0") testImplementation(libs.bundles.junit5) testImplementation(libs.bundles.mockito) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index f6936632f7c..0aa12723238 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -2,8 +2,8 @@ import static java.util.Arrays.asList; -import datadog.trace.api.config.FeatureFlaggingConfig; import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.config.FeatureFlaggingConfig; import datadog.trace.api.featureflag.exposure.ExposureEvent; import datadog.trace.api.featureflag.exposure.Subject; import datadog.trace.api.featureflag.ufc.v1.Allocation; diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index fe2159551d9..b65328f45ec 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -2,7 +2,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; -import datadog.trace.api.config.FeatureFlaggingConfig; +import datadog.trace.api.featureflag.config.FeatureFlaggingConfig; import datadog.trace.bootstrap.config.provider.ConfigProvider; import de.thetaphi.forbiddenapis.SuppressForbidden; import dev.openfeature.sdk.ErrorCode; diff --git a/products/feature-flagging/feature-flagging-config/build.gradle.kts b/products/feature-flagging/feature-flagging-config/build.gradle.kts new file mode 100644 index 00000000000..14109d8dfd9 --- /dev/null +++ b/products/feature-flagging/feature-flagging-config/build.gradle.kts @@ -0,0 +1,12 @@ +plugins { + `java-library` +} + +apply(from = "$rootDir/gradle/java.gradle") + +description = "Feature flagging configuration keys (compile-time constants)" + +extra["excludedClassesCoverage"] = listOf( + // Constants-only holder — no executable logic to cover. + "datadog.trace.api.featureflag.config.FeatureFlaggingConfig", +) diff --git a/products/feature-flagging/feature-flagging-config/gradle.lockfile b/products/feature-flagging/feature-flagging-config/gradle.lockfile new file mode 100644 index 00000000000..086d0fde3c0 --- /dev/null +++ b/products/feature-flagging/feature-flagging-config/gradle.lockfile @@ -0,0 +1,75 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +# To regenerate this file, run: ./gradlew :products:feature-flagging:feature-flagging-config:dependencies --write-locks +ch.qos.logback:logback-classic:1.2.13=testCompileClasspath,testRuntimeClasspath +ch.qos.logback:logback-core:1.2.13=testCompileClasspath,testRuntimeClasspath +com.github.javaparser:javaparser-core:3.25.6=codenarc +com.github.spotbugs:spotbugs-annotations:4.9.8=compileClasspath,spotbugs +com.github.spotbugs:spotbugs:4.9.8=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.code.findbugs:jsr305:3.0.2=compileClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +com.google.code.gson:gson:2.13.2=spotbugs +com.google.errorprone:error_prone_annotations:2.41.0=spotbugs +com.thoughtworks.qdox:qdox:1.12.1=codenarc +commons-io:commons-io:2.20.0=spotbugs +de.thetaphi:forbiddenapis:3.10=compileClasspath +io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath +jaxen:jaxen:2.0.0=spotbugs +net.bytebuddy:byte-buddy-agent:1.12.8=testRuntimeClasspath +net.bytebuddy:byte-buddy:1.12.8=testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.9=spotbugs +org.apache.ant:ant-antlr:1.10.14=codenarc +org.apache.ant:ant-junit:1.10.14=codenarc +org.apache.bcel:bcel:6.11.0=spotbugs +org.apache.commons:commons-lang3:3.19.0=spotbugs +org.apache.commons:commons-text:1.14.0=spotbugs +org.apache.logging.log4j:log4j-api:2.25.2=spotbugs +org.apache.logging.log4j:log4j-core:2.25.2=spotbugs +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testRuntimeClasspath +org.codehaus.groovy:groovy-ant:3.0.23=codenarc +org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc +org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.25=testCompileClasspath,testRuntimeClasspath +org.codehaus.groovy:groovy-templates:3.0.23=codenarc +org.codehaus.groovy:groovy-xml:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.25=testCompileClasspath,testRuntimeClasspath +org.codenarc:CodeNarc:3.7.0=codenarc +org.dom4j:dom4j:2.2.0=spotbugs +org.gmetrics:GMetrics:2.1.0=codenarc +org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath +org.jacoco:org.jacoco.agent:0.8.14=jacocoAgent,jacocoAnt +org.jacoco:org.jacoco.ant:0.8.14=jacocoAnt +org.jacoco:org.jacoco.core:0.8.14=jacocoAnt +org.jacoco:org.jacoco.report:0.8.14=jacocoAnt +org.junit.jupiter:junit-jupiter-api:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:5.14.1=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:5.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:1.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:1.14.1=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-launcher:1.14.1=testRuntimeClasspath +org.junit:junit-bom:5.14.0=spotbugs +org.junit:junit-bom:5.14.1=testCompileClasspath,testRuntimeClasspath +org.mockito:mockito-core:4.4.0=testRuntimeClasspath +org.objenesis:objenesis:3.3=testCompileClasspath,testRuntimeClasspath +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.9=spotbugs +org.ow2.asm:asm-commons:9.9=jacocoAnt,spotbugs +org.ow2.asm:asm-tree:9.9=jacocoAnt,spotbugs +org.ow2.asm:asm-util:9.9=spotbugs +org.ow2.asm:asm:9.9=jacocoAnt,spotbugs +org.slf4j:jcl-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:jul-to-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:log4j-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:1.7.32=testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:2.0.17=spotbugs,spotbugsSlf4j +org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j +org.spockframework:spock-bom:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath +org.spockframework:spock-core:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-junit:1.2.1=testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-parser:1.2.0=testCompileClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=spotbugs +empty=annotationProcessor,runtimeClasspath,spotbugsPlugins,testAnnotationProcessor diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java similarity index 93% rename from dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java rename to products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java index c915effde13..b32309c1fa7 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java @@ -1,4 +1,4 @@ -package datadog.trace.api.config; +package datadog.trace.api.featureflag.config; public class FeatureFlaggingConfig { diff --git a/settings.gradle.kts b/settings.gradle.kts index 36b1d9b95df..095376eaabc 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -161,6 +161,7 @@ include( ":products:feature-flagging:feature-flagging-agent", ":products:feature-flagging:feature-flagging-api", ":products:feature-flagging:feature-flagging-bootstrap", + ":products:feature-flagging:feature-flagging-config", ":products:feature-flagging:feature-flagging-lib" ) From 3b3ae4758022b34d5091f2c989b22b152ceb6749 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 9 Jul 2026 12:05:12 +0300 Subject: [PATCH 16/18] Fix 3 bugs: SpanEnrichmentWriter is now a process-wide singleton; addTraceInterceptor() false return: ensureInterceptorRegistered() now latches only on a true return; registration ordering: moved ensureInterceptorRegistered() below the root == null check --- .../featureflag/FeatureFlaggingSystem.java | 10 +-- .../featureflag/SpanEnrichmentWriter.java | 68 +++++++++++++------ .../featureflag/SpanEnrichmentWriterTest.java | 12 ++++ 3 files changed, 67 insertions(+), 23 deletions(-) diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index b6d77065780..9761caf3895 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -28,10 +28,12 @@ public static void start(final SharedCommunicationObjects sco) { EXPOSURE_WRITER = new ExposureWriterImpl(sco, config); EXPOSURE_WRITER.init(); - // APM span enrichment: agent-side listener for flag-evaluation seam events. Cheap to register - // (it only accumulates once the provider's gate-on capture hook actually dispatches events, and - // registers its trace interceptor lazily on the first such event). - SPAN_ENRICHMENT_WRITER = new SpanEnrichmentWriter(); + // APM span enrichment: agent-side listener for flag-evaluation seam events. Uses the process- + // wide singleton so a subsystem restart reuses the one already-registered trace interceptor + // (which the tracer cannot remove) instead of registering a second, rejected one. Cheap: it + // only accumulates once the provider's gate-on capture hook dispatches events, and registers + // its interceptor lazily on the first such event. + SPAN_ENRICHMENT_WRITER = SpanEnrichmentWriter.getInstance(); SPAN_ENRICHMENT_WRITER.init(); LOGGER.debug("Feature Flagging system started"); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java index 0430c8f34b2..cf4dbe474ce 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java @@ -14,22 +14,32 @@ * evaluation, resolves the active local-root span, and accumulates per-trace state that a {@link * SpanEnrichmentInterceptor} later flushes onto the root when the trace completes. * - *

    Living in the agent classloader (rather than the application-loaded provider) means the tracer - * API ({@link AgentSpan}/{@link AgentTracer}/{@link GlobalTracer}/{@code TraceInterceptor}) is used - * only here — the published product API never depends on {@code internal-api}. It also gives span - * enrichment a single, stable owner: no per-provider rebinding, no reconfiguration hazard, and no - * application-classloader pinning. + *

    Process-wide singleton (restart-safe). Use {@link #getInstance()} for the agent wiring. + * The tracer keeps trace interceptors for the life of the JVM and offers no removal API, so the + * interceptor — and the weak-keyed state it reads — must outlive any single start/stop of the + * feature-flagging subsystem. A fresh writer per {@code start()} would build a second interceptor at + * the same priority; the tracer would reject it and its state would never be read, silently + * disabling enrichment after a restart. Reusing one instance avoids that: the single interceptor is + * registered exactly once and simply resumes when {@link #init()} re-subscribes the listener. * *

    Zero idle overhead when off. When the span-enrichment gate is off the provider adds no * capture hook, so no seam events are dispatched, this listener never runs, and the interceptor is * never registered — the tracer's write path is untouched. The interceptor is registered lazily on - * the first enrichment event, so a service that enables the feature but never evaluates a flag on a - * traced request still pays nothing. + * the first enrichment event that has an active span, so a service that enables the feature but + * never evaluates a flag on a traced request still pays nothing. * *

    All work is wrapped in try/catch — enrichment must NEVER break flag evaluation. */ public final class SpanEnrichmentWriter implements FeatureFlaggingGateway.SpanEnrichmentListener { + // The one instance used by the agent. Persisting it across FeatureFlaggingSystem start/stop keeps + // the single registered interceptor (and its state) alive, so a restart never re-registers. + private static final SpanEnrichmentWriter INSTANCE = new SpanEnrichmentWriter(); + + public static SpanEnrichmentWriter getInstance() { + return INSTANCE; + } + /** * Resolves the local-root span for the active trace. Injectable so tests need no static mocks. */ @@ -47,26 +57,36 @@ interface RootSpanResolver { return localRoot != null ? localRoot : active; }; - private final SpanEnrichmentStates states = new SpanEnrichmentStates(); - private final SpanEnrichmentInterceptor interceptor = new SpanEnrichmentInterceptor(states); private final RootSpanResolver rootSpanResolver; - // Registered with the tracer at most once, lazily on the first enrichment event. + private final SpanEnrichmentStates states; + private final SpanEnrichmentInterceptor interceptor; + // Registered with the tracer at most once, lazily on the first enrichment event with an active + // span. Once true it stays true for the life of this instance, so a subsystem restart (which + // reuses the singleton) never attempts a second, doomed registration. private final AtomicBoolean interceptorRegistered = new AtomicBoolean(false); - public SpanEnrichmentWriter() { + private SpanEnrichmentWriter() { this(DEFAULT_RESOLVER); } + // Visible for tests: each test gets an isolated writer (own state + interceptor), bypassing the + // shared INSTANCE so tests never contaminate each other or the agent singleton. SpanEnrichmentWriter(final RootSpanResolver rootSpanResolver) { this.rootSpanResolver = rootSpanResolver; + this.states = new SpanEnrichmentStates(); + this.interceptor = new SpanEnrichmentInterceptor(states); } - /** Starts listening for enrichment events. */ + /** Starts listening for enrichment events. Safe to call again after {@link #close()}. */ public void init() { FeatureFlaggingGateway.addSpanEnrichmentListener(this); } - /** Stops listening and drops any residual state. */ + /** + * Stops listening and drops any residual state. The interceptor stays registered with the tracer + * (it cannot be removed) but goes inert while the state is empty; a later {@link #init()} resumes + * enrichment on the same interceptor. + */ public void close() { FeatureFlaggingGateway.removeSpanEnrichmentListener(this); states.clear(); @@ -78,11 +98,11 @@ public void accept(final SpanEnrichmentEvent event) { return; } try { - ensureInterceptorRegistered(); final AgentSpan root = rootSpanResolver.activeLocalRoot(); if (root == null) { - return; // no active span → nothing to enrich + return; // no active span → nothing to enrich (and nothing to register the interceptor for) } + ensureInterceptorRegistered(); final SpanEnrichmentAccumulator state = states.getOrCreate(root); if (event.hasSerialId()) { final int serialId = event.serialId(); @@ -99,12 +119,22 @@ public void accept(final SpanEnrichmentEvent event) { } private void ensureInterceptorRegistered() { - if (interceptorRegistered.compareAndSet(false, true)) { + if (interceptorRegistered.get()) { + return; + } + synchronized (this) { + if (interceptorRegistered.get()) { + return; + } try { - GlobalTracer.get().addTraceInterceptor(interceptor); + // addTraceInterceptor returns false (without throwing) when the tracer rejects it — e.g. the + // global tracer is still the no-op placeholder. Only latch on success so a later event + // retries; otherwise a transient false would permanently disable enrichment. + if (GlobalTracer.get().addTraceInterceptor(interceptor)) { + interceptorRegistered.set(true); + } } catch (final Throwable t) { - // Tracer not yet installed (e.g. no-op placeholder): allow a later event to retry. - interceptorRegistered.set(false); + // Leave unregistered; a later event retries. } } } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentWriterTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentWriterTest.java index 1f19ed187b3..259f785a2a2 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentWriterTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentWriterTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -102,6 +103,17 @@ void endToEndAccumulateThenFlush() { assertTrue(writer.states().isEmpty(), "flush removes the accumulated state"); } + @Test + void agentSingletonIsStableAcrossRestarts() { + // FeatureFlaggingSystem reuses this one instance across start/stop, so the (unremovable) trace + // interceptor is registered exactly once and never re-registered on restart. + final SpanEnrichmentWriter first = SpanEnrichmentWriter.getInstance(); + final SpanEnrichmentWriter second = SpanEnrichmentWriter.getInstance(); + assertSame(first, second, "agent wiring must reuse one writer across restarts"); + assertSame(first.interceptor(), second.interceptor(), "same interceptor across restarts"); + assertSame(first.states(), second.states(), "same state store across restarts"); + } + @Test void distinctEventsUnderSameRootShareAccumulator() { final AgentSpan root = rootSpan(); From f56665bae698a354b97ed4c29f48ad64f5aa2711 Mon Sep 17 00:00:00 2001 From: Pavel Date: Thu, 9 Jul 2026 12:18:05 +0300 Subject: [PATCH 17/18] Fix nits: add debug logging; fix comments --- .../trace/api/openfeature/DDEvaluator.java | 13 +------ .../trace/api/openfeature/Provider.java | 35 +++---------------- .../api/openfeature/SpanEnrichmentGate.java | 25 +++++++++++++ .../api/openfeature/SpanEnrichmentHook.java | 21 ++++++----- .../openfeature/SpanEnrichmentHookTest.java | 11 +++--- .../config/FeatureFlaggingConfig.java | 7 ++-- .../feature-flagging-lib/build.gradle.kts | 7 ++-- .../SpanEnrichmentInterceptor.java | 7 +++- .../featureflag/SpanEnrichmentWriter.java | 14 +++++--- 9 files changed, 66 insertions(+), 74 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentGate.java diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 0aa12723238..87f81ebab9f 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -3,7 +3,6 @@ import static java.util.Arrays.asList; import datadog.trace.api.featureflag.FeatureFlaggingGateway; -import datadog.trace.api.featureflag.config.FeatureFlaggingConfig; import datadog.trace.api.featureflag.exposure.ExposureEvent; import datadog.trace.api.featureflag.exposure.Subject; import datadog.trace.api.featureflag.ufc.v1.Allocation; @@ -17,7 +16,6 @@ import datadog.trace.api.featureflag.ufc.v1.Split; import datadog.trace.api.featureflag.ufc.v1.ValueType; import datadog.trace.api.featureflag.ufc.v1.Variant; -import datadog.trace.bootstrap.config.provider.ConfigProvider; import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.ImmutableMetadata; @@ -56,7 +54,7 @@ class DDEvaluator implements Evaluator, FeatureFlaggingGateway.ConfigListener { // Read once: when off, the __dd_* span-enrichment metadata is not attached to evaluations, so an // enabled provider pays nothing extra unless span enrichment is also enabled. The gate does not // change at runtime, and this class is loaded lazily (well after startup) so config is ready. - private static final boolean SPAN_ENRICHMENT_ENABLED = readSpanEnrichmentEnabled(); + private static final boolean SPAN_ENRICHMENT_ENABLED = SpanEnrichmentGate.isEnabled(); private final Runnable configCallback; private final AtomicReference configuration = new AtomicReference<>(); @@ -66,15 +64,6 @@ public DDEvaluator(final Runnable configCallback) { this.configCallback = configCallback; } - private static boolean readSpanEnrichmentEnabled() { - try { - return ConfigProvider.getInstance() - .getBoolean(FeatureFlaggingConfig.EXPERIMENTAL_SPAN_ENRICHMENT_ENABLED, false); - } catch (final Throwable t) { - return false; // never let config reading break evaluator class initialization - } - } - @Override public boolean initialize( final long timeout, final TimeUnit unit, final EvaluationContext context) throws Exception { diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index b65328f45ec..38fa735d4e9 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -2,8 +2,6 @@ import static java.util.concurrent.TimeUnit.SECONDS; -import datadog.trace.api.featureflag.config.FeatureFlaggingConfig; -import datadog.trace.bootstrap.config.provider.ConfigProvider; import de.thetaphi.forbiddenapis.SuppressForbidden; import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.EvaluationContext; @@ -32,17 +30,6 @@ public class Provider extends EventProvider implements Metadata { static final String METADATA = "datadog-openfeature-provider"; private static final String EVALUATOR_IMPL = "datadog.trace.api.openfeature.DDEvaluator"; - /** - * Canonical config key for the span-enrichment gate ({@link - * FeatureFlaggingConfig#EXPERIMENTAL_SPAN_ENRICHMENT_ENABLED}). Read through {@link - * ConfigProvider} so the full precedence applies — system property, env var ({@code - * DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED}), and stable config — exactly like - * the sibling provider-enabled gate. Distinct from the provider-enabled gate; OFF by default - * (experimental opt-in). - */ - static final String SPAN_ENRICHMENT_ENABLED_KEY = - FeatureFlaggingConfig.EXPERIMENTAL_SPAN_ENRICHMENT_ENABLED; - private static final Options DEFAULT_OPTIONS = new Options().initTimeout(30, SECONDS); private volatile Evaluator evaluator; private final Options options; @@ -70,7 +57,7 @@ public Provider(final Options options) { /** * @param spanEnrichmentEnabledOverride when non-null, forces the span-enrichment gate (test - * seam); when null, the gate is read from {@link #SPAN_ENRICHMENT_ENABLED_KEY}. + * seam); when null, the gate is read via {@link SpanEnrichmentGate}. */ Provider( final Options options, @@ -90,14 +77,12 @@ public Provider(final Options options) { this.flagEvalMetrics = metrics; this.flagEvalHook = hook; - // Span enrichment is wired ONLY when the gate is on. When off, no capture hook is constructed - // and there is no idle per-evaluation overhead. The hook merely dispatches evaluation metadata - // onto FeatureFlaggingGateway; the agent-side write tier (feature-flagging-lib) resolves the - // span and accumulates, so this application-side provider holds no tracer dependency. + // Span enrichment is wired ONLY when the gate is on — off means no capture hook and no idle + // per-evaluation overhead. final boolean spanEnrichmentEnabled = spanEnrichmentEnabledOverride != null ? spanEnrichmentEnabledOverride - : isSpanEnrichmentEnabled(); + : SpanEnrichmentGate.isEnabled(); this.spanEnrichmentHook = spanEnrichmentEnabled ? new SpanEnrichmentHook() : null; // Precompute the immutable hook list once so getProviderHooks() (called on every evaluation) @@ -121,16 +106,6 @@ public Provider(final Options options) { } } - private static boolean isSpanEnrichmentEnabled() { - try { - // Full config precedence (system property > stable config > env) via ConfigProvider, matching - // the sibling provider-enabled gate. "1"/"true" (any case) map to true; default false. - return ConfigProvider.getInstance().getBoolean(SPAN_ENRICHMENT_ENABLED_KEY, false); - } catch (final Throwable t) { - return false; // never let config reading break provider construction - } - } - @Override public void initialize(final EvaluationContext context) throws Exception { initializationState.set(InitializationState.INITIALIZING); @@ -299,7 +274,7 @@ public ProviderEvaluation getObjectEvaluation( return evaluator.evaluate(Value.class, key, defaultValue, ctx); } - @SuppressForbidden // Class#forName(String) used to lazy load internal-api dependencies + @SuppressForbidden // Class#forName(String) used to lazy-load the evaluator implementation protected Class loadEvaluatorClass() throws ClassNotFoundException { return Class.forName(EVALUATOR_IMPL); } diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentGate.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentGate.java new file mode 100644 index 00000000000..64739f0ad39 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentGate.java @@ -0,0 +1,25 @@ +package datadog.trace.api.openfeature; + +import datadog.trace.api.featureflag.config.FeatureFlaggingConfig; +import datadog.trace.bootstrap.config.provider.ConfigProvider; + +/** + * Single source for reading the experimental span-enrichment gate, with full {@link ConfigProvider} + * precedence (system property > stable config > env var {@code + * DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED}). OFF by default; distinct from the + * provider-enabled gate. Shared so {@link Provider} (per construction) and {@link DDEvaluator} + * (once at class load) read it the same way. + */ +final class SpanEnrichmentGate { + + private SpanEnrichmentGate() {} + + static boolean isEnabled() { + try { + return ConfigProvider.getInstance() + .getBoolean(FeatureFlaggingConfig.EXPERIMENTAL_SPAN_ENRICHMENT_ENABLED, false); + } catch (final Throwable t) { + return false; // never let config reading break construction + } + } +} diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java index b93d249c2b1..351a1658efe 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java @@ -14,19 +14,15 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * OpenFeature {@code finally} hook that captures feature-flag evaluation metadata for APM span - * enrichment. This is the CAPTURE half of the capture-vs-write split, and it runs in the - * application classloader (returned from {@link Provider#getProviderHooks()} only when the gate is - * on). - * - *

    It holds no tracer dependency: rather than resolving the active span itself, it emits a - * {@link SpanEnrichmentEvent} onto {@link FeatureFlaggingGateway}. The agent-side write tier - * ({@code SpanEnrichmentWriter} in {@code feature-flagging-lib}) receives the event, resolves the - * active local-root span, and accumulates the per-trace state that is flushed onto the root when - * the trace completes. This keeps the published {@code dd-openfeature} product API free of {@code - * internal-api}. + * enrichment. Registered from {@link Provider#getProviderHooks()} only when the gate is on. For + * each relevant evaluation it dispatches a {@link SpanEnrichmentEvent} onto {@link + * FeatureFlaggingGateway}; the agent-side write tier resolves the active local-root span and + * accumulates the per-trace state that is flushed onto the root when the trace completes. * *

    Capture branch (frozen Node reference): * @@ -41,6 +37,8 @@ */ class SpanEnrichmentHook implements Hook { + private static final Logger log = LoggerFactory.getLogger(SpanEnrichmentHook.class); + // The metadata keys the DDEvaluator attaches for span enrichment (single source of truth there). static final String METADATA_SERIAL_ID = DDEvaluator.METADATA_SPLIT_SERIAL_ID; static final String METADATA_DO_LOG = DDEvaluator.METADATA_DO_LOG; @@ -75,7 +73,8 @@ public void finallyAfter( details.getFlagKey(), unwrapDefaultValue(details.getValue()))); } } catch (final Throwable t) { - // Never let span enrichment break flag evaluation. + // Never let span enrichment break flag evaluation; a debug line aids diagnosis if it does. + log.debug("Span-enrichment capture failed for flag {}", details.getFlagKey(), t); } } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java index 3ed05b94abd..400381a9a8a 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java @@ -27,13 +27,10 @@ import org.junit.jupiter.api.Test; /** - * Capture-side unit suite for APM feature-flag span enrichment. - * - *

    The published {@code dd-openfeature} provider has no tracer dependency: the hook dispatches a - * {@link SpanEnrichmentEvent} onto {@link FeatureFlaggingGateway} (native JDK types only) and the - * agent-side write tier does the accumulation. These tests therefore assert the dispatched events - * (not span tags — those are covered in {@code feature-flagging-lib}) plus the {@link Provider} - * gating. + * Capture-side unit suite for APM feature-flag span enrichment. The hook dispatches a {@link + * SpanEnrichmentEvent} onto {@link FeatureFlaggingGateway} and the agent-side write tier does the + * accumulation, so these tests assert the dispatched events (not span tags — those are covered in + * {@code feature-flagging-lib}) plus the {@link Provider} gating. */ class SpanEnrichmentHookTest { diff --git a/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java b/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java index b32309c1fa7..9d75961e9d4 100644 --- a/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java +++ b/products/feature-flagging/feature-flagging-config/src/main/java/datadog/trace/api/featureflag/config/FeatureFlaggingConfig.java @@ -5,10 +5,9 @@ public class FeatureFlaggingConfig { public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; /** - * Opt-in gate for APM span enrichment with feature-flag evaluation metadata. The dot-form maps to - * {@code DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED} via the dot-to-underscore + - * {@code DD_} prefix normalization rule. This is DISTINCT from {@link #FLAGGING_PROVIDER_ENABLED} - * and is OFF by default — enabling the provider does not enable span enrichment. + * Opt-in gate for APM span enrichment with feature-flag evaluation metadata. DISTINCT from {@link + * #FLAGGING_PROVIDER_ENABLED} and OFF by default — enabling the provider does not enable span + * enrichment. */ public static final String EXPERIMENTAL_SPAN_ENRICHMENT_ENABLED = "experimental.flagging.provider.span.enrichment.enabled"; diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 2008af2b675..9d659d6ed63 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -22,12 +22,9 @@ dependencies { api(project(":utils:queue-utils")) compileOnly(project(":dd-trace-core")) // shading does not work with this one - // Span-enrichment write tier: TraceInterceptor / GlobalTracer / AgentTracer / AgentSpan. This is - // an agent-side module (shaded into the agent, never published), so depending on the tracer's - // internal API is expected — unlike the published dd-openfeature product API. + // Span-enrichment write tier: TraceInterceptor / GlobalTracer / AgentTracer / AgentSpan. compileOnly(project(":internal-api")) - // Platform JSON writer for the ffe_* tag values (provided by the agent at runtime, like the - // tracer API above; also reachable transitively via :internal-api). + // Platform JSON writer for the ffe_* tag values. compileOnly(project(":components:json")) testImplementation(libs.bundles.junit5) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentInterceptor.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentInterceptor.java index 7f3cf0d5ff0..672d884d30f 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentInterceptor.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentInterceptor.java @@ -5,6 +5,8 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import java.util.Collection; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * {@link TraceInterceptor} that writes the accumulated {@code ffe_*} tags onto the local root span @@ -44,6 +46,8 @@ */ final class SpanEnrichmentInterceptor implements TraceInterceptor { + private static final Logger log = LoggerFactory.getLogger(SpanEnrichmentInterceptor.class); + /** * Unique priority in the "trace data enrichment" band, after {@code GIT_METADATA} (3) and before * the custom-sampling band ({@code Integer.MAX_VALUE - 2}). Distinct from every value in {@code @@ -85,7 +89,8 @@ public Collection onTraceComplete( } } } catch (final Throwable t) { - // Never let span enrichment break trace finish. + // Never let span enrichment break trace finish; a debug line aids diagnosis if it does. + log.debug("Span-enrichment tag write failed", t); } return trace; } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java index cf4dbe474ce..708a96794cb 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java @@ -6,6 +6,8 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Agent-side owner of APM feature-flag span enrichment. This is the WRITE tier of the @@ -17,8 +19,8 @@ *

    Process-wide singleton (restart-safe). Use {@link #getInstance()} for the agent wiring. * The tracer keeps trace interceptors for the life of the JVM and offers no removal API, so the * interceptor — and the weak-keyed state it reads — must outlive any single start/stop of the - * feature-flagging subsystem. A fresh writer per {@code start()} would build a second interceptor at - * the same priority; the tracer would reject it and its state would never be read, silently + * feature-flagging subsystem. A fresh writer per {@code start()} would build a second interceptor + * at the same priority; the tracer would reject it and its state would never be read, silently * disabling enrichment after a restart. Reusing one instance avoids that: the single interceptor is * registered exactly once and simply resumes when {@link #init()} re-subscribes the listener. * @@ -32,6 +34,8 @@ */ public final class SpanEnrichmentWriter implements FeatureFlaggingGateway.SpanEnrichmentListener { + private static final Logger log = LoggerFactory.getLogger(SpanEnrichmentWriter.class); + // The one instance used by the agent. Persisting it across FeatureFlaggingSystem start/stop keeps // the single registered interceptor (and its state) alive, so a restart never re-registers. private static final SpanEnrichmentWriter INSTANCE = new SpanEnrichmentWriter(); @@ -114,7 +118,8 @@ public void accept(final SpanEnrichmentEvent event) { state.addDefault(event.flagKey(), event.defaultValue()); } } catch (final Throwable t) { - // Never let span enrichment break flag evaluation. + // Never let span enrichment break flag evaluation; a debug line aids diagnosis if it does. + log.debug("Span-enrichment accumulation failed", t); } } @@ -127,7 +132,8 @@ private void ensureInterceptorRegistered() { return; } try { - // addTraceInterceptor returns false (without throwing) when the tracer rejects it — e.g. the + // addTraceInterceptor returns false (without throwing) when the tracer rejects it — e.g. + // the // global tracer is still the no-op placeholder. Only latch on success so a later event // retries; otherwise a transient false would permanently disable enrichment. if (GlobalTracer.get().addTraceInterceptor(interceptor)) { From d3b867a027329d7aa24400b0a40b173c6ef18c9b Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 10 Jul 2026 11:43:15 +0300 Subject: [PATCH 18/18] Add gate on registration; adjust - add typed metadata --- .../trace/api/openfeature/DDEvaluator.java | 5 +- .../api/openfeature/SpanEnrichmentHook.java | 13 ++--- .../openfeature/SpanEnrichmentHookTest.java | 11 ++-- .../featureflag/SpanEnrichmentWriter.java | 50 ++++++++++++++----- .../featureflag/SpanEnrichmentWriterTest.java | 10 ++++ 5 files changed, 59 insertions(+), 30 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 87f81ebab9f..654fb6e8f6c 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -409,10 +409,9 @@ private static ProviderEvaluation resolveVariant( // subject. if (SPAN_ENRICHMENT_ENABLED) { if (split.serialId != null) { - metadataBuilder.addString(METADATA_SPLIT_SERIAL_ID, split.serialId.toString()); + metadataBuilder.addInteger(METADATA_SPLIT_SERIAL_ID, split.serialId); } - metadataBuilder.addString( - METADATA_DO_LOG, String.valueOf(allocation.doLog != null && allocation.doLog)); + metadataBuilder.addBoolean(METADATA_DO_LOG, allocation.doLog != null && allocation.doLog); } final ProviderEvaluation result = ProviderEvaluation.builder() diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java index 351a1658efe..40e998a6546 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/SpanEnrichmentHook.java @@ -53,16 +53,9 @@ public void finallyAfter( } try { final ImmutableMetadata metadata = details.getFlagMetadata(); - final String serialIdStr = metadata != null ? metadata.getString(METADATA_SERIAL_ID) : null; - if (serialIdStr != null) { - final int serialId; - try { - serialId = Integer.parseInt(serialIdStr); - } catch (final NumberFormatException e) { - return; // malformed serial id — drop, never break eval - } - final String doLogStr = metadata.getString(METADATA_DO_LOG); - final boolean doLog = "true".equalsIgnoreCase(doLogStr); + final Integer serialId = metadata != null ? metadata.getInteger(METADATA_SERIAL_ID) : null; + if (serialId != null) { + final boolean doLog = Boolean.TRUE.equals(metadata.getBoolean(METADATA_DO_LOG)); FeatureFlaggingGateway.dispatch( SpanEnrichmentEvent.serialId(serialId, doLog, targetingKey(ctx))); } else if (details.getVariant() == null) { diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java index 400381a9a8a..2ce0931e116 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/SpanEnrichmentHookTest.java @@ -65,9 +65,9 @@ private static FlagEvaluationDetails details( private static ImmutableMetadata metadata(final Integer serialId, final boolean doLog) { final ImmutableMetadata.ImmutableMetadataBuilder builder = ImmutableMetadata.builder(); if (serialId != null) { - builder.addString(SpanEnrichmentHook.METADATA_SERIAL_ID, serialId.toString()); + builder.addInteger(SpanEnrichmentHook.METADATA_SERIAL_ID, serialId); } - builder.addString(SpanEnrichmentHook.METADATA_DO_LOG, String.valueOf(doLog)); + builder.addBoolean(SpanEnrichmentHook.METADATA_DO_LOG, doLog); return builder.build(); } @@ -116,16 +116,17 @@ void serialIdWithoutDoLogStillDispatchesSerialButNotDoLog() { } @Test - void malformedSerialIdDispatchesNothing() { + void wrongTypedSerialIdDispatchesNothing() { + // Defensive: a non-integer value under the serial-id key (wrong type) is ignored, not crashed. final ImmutableMetadata bad = ImmutableMetadata.builder() .addString(SpanEnrichmentHook.METADATA_SERIAL_ID, "not-a-number") - .addString(SpanEnrichmentHook.METADATA_DO_LOG, "true") + .addBoolean(SpanEnrichmentHook.METADATA_DO_LOG, true) .build(); new SpanEnrichmentHook() .finallyAfter( ctx("flag", "user-1"), details("flag", "on", "v", bad), Collections.emptyMap()); - assertTrue(captured.isEmpty(), "a malformed serial id must never break eval or dispatch"); + assertTrue(captured.isEmpty(), "a wrong-typed serial id must never break eval or dispatch"); } // ---- runtime-default branch (missing variant) ---- diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java index 708a96794cb..96c75074edb 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/SpanEnrichmentWriter.java @@ -51,6 +51,14 @@ interface RootSpanResolver { AgentSpan activeLocalRoot(); } + /** + * Registers the interceptor with the tracer, returning {@code true} when accepted. Injectable so + * tests are deterministic without a globally-installed tracer. + */ + interface InterceptorRegistrar { + boolean register(SpanEnrichmentInterceptor interceptor); + } + private static final RootSpanResolver DEFAULT_RESOLVER = () -> { final AgentSpan active = AgentTracer.activeSpan(); @@ -61,7 +69,11 @@ interface RootSpanResolver { return localRoot != null ? localRoot : active; }; + private static final InterceptorRegistrar DEFAULT_REGISTRAR = + interceptor -> GlobalTracer.get().addTraceInterceptor(interceptor); + private final RootSpanResolver rootSpanResolver; + private final InterceptorRegistrar registrar; private final SpanEnrichmentStates states; private final SpanEnrichmentInterceptor interceptor; // Registered with the tracer at most once, lazily on the first enrichment event with an active @@ -70,13 +82,20 @@ interface RootSpanResolver { private final AtomicBoolean interceptorRegistered = new AtomicBoolean(false); private SpanEnrichmentWriter() { - this(DEFAULT_RESOLVER); + this(DEFAULT_RESOLVER, DEFAULT_REGISTRAR); } - // Visible for tests: each test gets an isolated writer (own state + interceptor), bypassing the - // shared INSTANCE so tests never contaminate each other or the agent singleton. + // Visible for tests: an isolated writer (own state + interceptor) whose interceptor registration + // is assumed to succeed, bypassing the shared INSTANCE and any globally-installed tracer. SpanEnrichmentWriter(final RootSpanResolver rootSpanResolver) { + this(rootSpanResolver, interceptor -> true); + } + + // Visible for tests: also inject the registrar to exercise the not-registered path. + SpanEnrichmentWriter( + final RootSpanResolver rootSpanResolver, final InterceptorRegistrar registrar) { this.rootSpanResolver = rootSpanResolver; + this.registrar = registrar; this.states = new SpanEnrichmentStates(); this.interceptor = new SpanEnrichmentInterceptor(states); } @@ -106,7 +125,11 @@ public void accept(final SpanEnrichmentEvent event) { if (root == null) { return; // no active span → nothing to enrich (and nothing to register the interceptor for) } - ensureInterceptorRegistered(); + if (!ensureInterceptorRegistered()) { + // The interceptor isn't registered (e.g. tracer absent), so nothing would ever flush this + // state — skip accumulating. A later event retries registration. + return; + } final SpanEnrichmentAccumulator state = states.getOrCreate(root); if (event.hasSerialId()) { final int serialId = event.serialId(); @@ -123,25 +146,28 @@ public void accept(final SpanEnrichmentEvent event) { } } - private void ensureInterceptorRegistered() { + /** + * @return true once the interceptor is registered with the tracer. + */ + private boolean ensureInterceptorRegistered() { if (interceptorRegistered.get()) { - return; + return true; } synchronized (this) { if (interceptorRegistered.get()) { - return; + return true; } try { - // addTraceInterceptor returns false (without throwing) when the tracer rejects it — e.g. - // the - // global tracer is still the no-op placeholder. Only latch on success so a later event - // retries; otherwise a transient false would permanently disable enrichment. - if (GlobalTracer.get().addTraceInterceptor(interceptor)) { + // register() returns false (without throwing) when the tracer rejects it — e.g. the global + // tracer is still the no-op placeholder. Only latch on success so a later event retries; + // otherwise a transient false would permanently disable enrichment. + if (registrar.register(interceptor)) { interceptorRegistered.set(true); } } catch (final Throwable t) { // Leave unregistered; a later event retries. } + return interceptorRegistered.get(); } } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentWriterTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentWriterTest.java index 259f785a2a2..c45ff6b2e66 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentWriterTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/SpanEnrichmentWriterTest.java @@ -103,6 +103,16 @@ void endToEndAccumulateThenFlush() { assertTrue(writer.states().isEmpty(), "flush removes the accumulated state"); } + @Test + void doesNotAccumulateWhenInterceptorCannotRegister() { + final AgentSpan root = rootSpan(); + // Registrar always rejects → nothing would ever flush the state, so we must not accumulate. + final SpanEnrichmentWriter writer = new SpanEnrichmentWriter(() -> root, interceptor -> false); + writer.accept(SpanEnrichmentEvent.serialId(5, false, null)); + assertTrue( + writer.states().isEmpty(), "no accumulation when the interceptor cannot be registered"); + } + @Test void agentSingletonIsStableAcrossRestarts() { // FeatureFlaggingSystem reuses this one instance across start/stop, so the (unremovable) trace