Repository navigation
Send EXPOSURE and EVALUATION feature flag features to Datadog #12797
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
d37ceb0
01e1e52
4986c3a
c1bfc0a
fb27c9c
49a9c18
bd4c417
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -68,6 +68,27 @@ class DDEvaluator implements Evaluator, FeatureFlaggingGateway.ConfigListener { | |
| static final AtomicBoolean SPLIT_FEATURES_SUPPORTED = | ||
| new AtomicBoolean(splitFeaturesSupported(Split.class)); | ||
|
|
||
| static final AtomicBoolean EXPOSURE_FEATURES_SUPPORTED = | ||
| new AtomicBoolean(exposureFeaturesSupported(ExposureEvent.class)); | ||
|
|
||
| /** An older agent's exposure event has no features; its exposures are then sent without them. */ | ||
| static boolean exposureFeaturesSupported(final Class<?> eventClass) { | ||
| try { | ||
| eventClass.getConstructor( | ||
| long.class, | ||
| datadog.trace.api.featureflag.exposure.Allocation.class, | ||
| datadog.trace.api.featureflag.exposure.Flag.class, | ||
| datadog.trace.api.featureflag.exposure.Variant.class, | ||
| Subject.class, | ||
| Integer.class, | ||
| Map.class); | ||
| return true; | ||
| } catch (final NoSuchMethodException | LinkageError | RuntimeException e) { | ||
| log.debug("The installed Datadog Java agent does not send features on exposures", e); | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| /** An older agent ships a Split without features; its evaluations then carry no features. */ | ||
| static boolean splitFeaturesSupported(final Class<?> splitClass) { | ||
| try { | ||
|
|
@@ -164,12 +185,19 @@ static boolean exposureSerialIdSupported(final Class<?> eventClass) { | |
| // Emitted only when the allocation logs exposures: true when this subject's exposure was already | ||
| // sent, so the exposure hook does not send it again. | ||
| static final String METADATA_EXPOSURE_CACHE_HIT = "__dd_exposure_cache_hit"; | ||
| // Each feature of the selected split is copied to the metadata under this prefix. | ||
| // Each feature of the selected split is copied to the metadata once for each destination it | ||
| // lists, under that destination's prefix. Only the HOOK prefix is a public contract; the | ||
| // other two carry features to Datadog's own exposure and flag-evaluation hooks. | ||
| static final String METADATA_FEATURE_PREFIX = "__dd_feature."; | ||
| static final String METADATA_EXPOSURE_FEATURE_PREFIX = "__dd_exposure_feature."; | ||
| static final String METADATA_EVALUATION_FEATURE_PREFIX = "__dd_evaluation_feature."; | ||
| // Present only when the split routes a feature to EVALUATION, so FlagEvalLoggingHook can skip | ||
| // scanning the metadata on every other evaluation. | ||
| static final String METADATA_HAS_EVALUATION_FEATURES = "__dd_has_evaluation_features"; | ||
|
|
||
| static final String FEATURE_DESTINATION_HOOK = "HOOK"; | ||
| private static final Set<String> KNOWN_FEATURE_DESTINATIONS = | ||
| new HashSet<>(asList(FEATURE_DESTINATION_HOOK, "EXPOSURE", "EVALUATION")); | ||
| static final String FEATURE_DESTINATION_EXPOSURE = "EXPOSURE"; | ||
| static final String FEATURE_DESTINATION_EVALUATION = "EVALUATION"; | ||
| // A newer UFC can name destinations this SDK does not know; warn once per name, not per | ||
| // evaluation. | ||
| static final int MAX_WARNED_FEATURE_DESTINATIONS = 64; | ||
|
|
@@ -669,40 +697,62 @@ private static void addSplitFeatures( | |
| if (split.features == null) { | ||
| return; | ||
| } | ||
| Set<String> seenKeys = null; | ||
| boolean evaluationFeatures = false; | ||
| for (final Object item : split.features) { | ||
| if (!(item instanceof Feature)) { | ||
| continue; | ||
| } | ||
| final Feature feature = (Feature) item; | ||
| if (feature.key == null || !deliversToHook(feature)) { | ||
| if (feature.key == null || feature.destinations == null) { | ||
| continue; | ||
| } | ||
| final String key = METADATA_FEATURE_PREFIX + feature.key; | ||
| if (feature.value instanceof String) { | ||
| metadataBuilder.addString(key, (String) feature.value); | ||
| } else if (feature.value instanceof Boolean) { | ||
| metadataBuilder.addBoolean(key, (Boolean) feature.value); | ||
| } else if (feature.value instanceof Number) { | ||
| metadataBuilder.addDouble(key, ((Number) feature.value).doubleValue()); | ||
| if (seenKeys == null) { | ||
| seenKeys = new HashSet<>(); | ||
| } | ||
| if (!seenKeys.add(feature.key)) { | ||
| continue; | ||
| } | ||
| for (final String destination : feature.destinations) { | ||
| final String prefix = metadataPrefix(destination); | ||
| if (prefix != null) { | ||
| addFeature(metadataBuilder, prefix + feature.key, feature.value); | ||
| evaluationFeatures |= prefix.equals(METADATA_EVALUATION_FEATURE_PREFIX); | ||
| } else if (WARNED_FEATURE_DESTINATIONS.size() < MAX_WARNED_FEATURE_DESTINATIONS | ||
| && WARNED_FEATURE_DESTINATIONS.add(destination)) { | ||
| log.warn("Ignoring unknown feature destination {}", destination); | ||
| } | ||
| } | ||
| } | ||
| if (evaluationFeatures) { | ||
| metadataBuilder.addBoolean(METADATA_HAS_EVALUATION_FEATURES, true); | ||
| } | ||
| } | ||
|
|
||
| private static boolean deliversToHook(final Feature feature) { | ||
| if (feature.destinations == null) { | ||
| return false; | ||
| private static String metadataPrefix(final String destination) { | ||
| if (FEATURE_DESTINATION_HOOK.equals(destination)) { | ||
| return METADATA_FEATURE_PREFIX; | ||
| } | ||
| boolean hook = false; | ||
| for (final String destination : feature.destinations) { | ||
| if (FEATURE_DESTINATION_HOOK.equals(destination)) { | ||
| hook = true; | ||
| } else if (!KNOWN_FEATURE_DESTINATIONS.contains(destination) | ||
| && WARNED_FEATURE_DESTINATIONS.size() < MAX_WARNED_FEATURE_DESTINATIONS | ||
| && WARNED_FEATURE_DESTINATIONS.add(destination)) { | ||
| log.warn("Ignoring unknown feature destination {}", destination); | ||
| } | ||
| if (FEATURE_DESTINATION_EXPOSURE.equals(destination)) { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🤖 Comment from Claude working with Aaron Silverman: Reminder: release this only after ddoghq/dd-source#124673 is deployed. Until then, a flag configuration that routes a feature to EXPOSURE makes intake drop whole exposure batches, and one that routes a feature to EVALUATION does the same to flag-evaluation batches. The intake schemas reject unknown fields, and #124673 is still open. Each worker decodes an Evidence — dd-source main
|
||
| return METADATA_EXPOSURE_FEATURE_PREFIX; | ||
| } | ||
| if (FEATURE_DESTINATION_EVALUATION.equals(destination)) { | ||
| return METADATA_EVALUATION_FEATURE_PREFIX; | ||
| } | ||
| return null; | ||
| } | ||
|
|
||
| private static void addFeature( | ||
| final ImmutableMetadata.ImmutableMetadataBuilder metadataBuilder, | ||
| final String key, | ||
| final Object value) { | ||
| if (value instanceof String) { | ||
| metadataBuilder.addString(key, (String) value); | ||
| } else if (value instanceof Boolean) { | ||
| metadataBuilder.addBoolean(key, (Boolean) value); | ||
| } else if (value instanceof Number) { | ||
| metadataBuilder.addDouble(key, ((Number) value).doubleValue()); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🤖 Comment from Claude working with Aaron Silverman: [P2] Settle the type of The bandit RFC carries the policy id as a split feature, and the RFC does not say whether it is a string or a number. The agent's parser turns every JSON number in a feature into a Evidence — measured at this PR's head
Suggested fixHave the UFC compiler emit
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. A string policy id avoids this. I will raise it on the bandit RFC. This was replied to by Claude. |
||
| } | ||
| return hook; | ||
| } | ||
|
|
||
| /** The serial id an exposure carries: none when the agent's exposure event predates it. */ | ||
|
|
@@ -784,7 +834,8 @@ static ExposureEvent exposureEvent( | |
| final String allocationKey, | ||
| final String variantKey, | ||
| final EvaluationContext context, | ||
| final Integer serialId) { | ||
| final Integer serialId, | ||
| final Map<String, Object> features) { | ||
| final long timestamp = System.currentTimeMillis(); | ||
| // Exposure types share names with the imported UFC Allocation, Flag, and Variant types. | ||
| final datadog.trace.api.featureflag.exposure.Allocation allocation = | ||
|
|
@@ -795,9 +846,14 @@ static ExposureEvent exposureEvent( | |
| new datadog.trace.api.featureflag.exposure.Variant(variantKey); | ||
| final Subject subject = new Subject(context.getTargetingKey(), flattenContext(context)); | ||
|
|
||
| return EXPOSURE_SERIAL_ID_SUPPORTED.get() | ||
| ? new ExposureEvent(timestamp, allocation, exposureFlag, variant, subject, serialId) | ||
| : new ExposureEvent(timestamp, allocation, exposureFlag, variant, subject); | ||
| if (!EXPOSURE_SERIAL_ID_SUPPORTED.get()) { | ||
| return new ExposureEvent(timestamp, allocation, exposureFlag, variant, subject); | ||
| } | ||
| if (features.isEmpty() || !EXPOSURE_FEATURES_SUPPORTED.get()) { | ||
| return new ExposureEvent(timestamp, allocation, exposureFlag, variant, subject, serialId); | ||
| } | ||
| return new ExposureEvent( | ||
| timestamp, allocation, exposureFlag, variant, subject, serialId, features); | ||
| } | ||
|
|
||
| static AbstractMap<String, Object> flattenContext(final EvaluationContext context) { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| package datadog.trace.api.openfeature; | ||
|
|
||
| import dev.openfeature.sdk.ImmutableMetadata; | ||
| import java.util.Collections; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
|
|
||
| /** | ||
| * Reads the split features that {@link DDEvaluator} copies into flag metadata. It depends only on | ||
| * the OpenFeature API, so {@link ExposureHook} keeps working where the Datadog Java agent is | ||
| * absent. | ||
| */ | ||
| final class SplitFeatureMetadata { | ||
|
|
||
| private SplitFeatureMetadata() {} | ||
|
|
||
| /** The features stored in the metadata under {@code prefix}, keyed without the prefix. */ | ||
| static Map<String, Object> featuresWithPrefix( | ||
| final ImmutableMetadata metadata, final String prefix) { | ||
| if (metadata == null) { | ||
| return Collections.emptyMap(); | ||
| } | ||
| Map<String, Object> features = null; | ||
| for (final Map.Entry<String, Object> entry : metadata.asUnmodifiableMap().entrySet()) { | ||
| if (entry.getKey().startsWith(prefix)) { | ||
| if (features == null) { | ||
| features = new HashMap<>(); | ||
| } | ||
| features.put(entry.getKey().substring(prefix.length()), entry.getValue()); | ||
| } | ||
| } | ||
| return features == null | ||
| ? Collections.<String, Object>emptyMap() | ||
| : Collections.unmodifiableMap(features); | ||
| } | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🤖 Comment from Claude working with Aaron Silverman:
[P2] Move the agent-capability probes into one class, and note on each the
dd-java-agentversion that added the capability. This stack takes them from two to five, spread overDDEvaluatorandFlagEvalLoggingHook, and none of them names the version it depends on.One class shows the version skew in one place, and makes each probe easy to delete when
dd-openfeatureraises its minimum agent version. The class must keepFlagEvalEventout of its own static initializer. On an agent before 1.66.0 that would throwNoClassDefFoundErrorand breakDDEvaluator, which is the bug thatbc1cdd4502fixed. A nested holder class that onlyFlagEvalLoggingHookloads avoids it, and a test pins it.Suggested change —
AgentSupport, on top of the enum from my previous comment (tested)This diff is relative to the previous suggestion. The other four test files only change their references from
DDEvaluator.*andFlagEvalLoggingHook.*toAgentSupport.*; they are left out here.Evidence — tests, two mutants, and the compiled classes
:products:feature-flagging:feature-flagging-api:testpasses 517 tests,forkedTestpasses 4, andspotlessCheckpasses.AgentSupportTestloads the provider classes through a class loader that hidesFlagEvalEvent, the class that agents before 1.66.0 lack. It covers that missing class only, not every other difference in an older agent.DDEvaluatorandAgentSupportstill initialize, andEXPOSURE_EVENT_APIisWITH_FEATURES. InitializingAgentSupport$FlagEvalEventsin that loader throwsNoClassDefFoundError, as a positive control.FlagEvalEventprobe added toAgentSupport's static initializer makes the test fail withNoClassDefFoundErroratAgentSupport.<clinit>.DDEvaluator's static initializer, the original bug, makes it fail atDDEvaluator.<clinit>.javapon the compiled classes:AgentSupport.classandDDEvaluator.classhave no reference toFlagEvalEvent.AgentSupport$FlagEvalEvents.classhas 3.Two behavior changes:
AgentSupportfield. For theSplitandExposureEventprobes, that is the first evaluation that resolves a variant, or the first exposure built; before, they ran whenDDEvaluatorloaded. For theFlagEvalEventprobe, it is the first timeFlagEvalLoggingHookbuilds a row that has EVALUATION features; before, it ran when the hook loaded. So on an older agent, the serial-id warning appears at the first resolved evaluation, not at startup. To keep the old timing, read anAgentSupportfield inDDEvaluator's static initializer, and anAgentSupport.FlagEvalEventsfield inFlagEvalLoggingHook's.AgentSupportlogger instead of theDDEvaluatorandFlagEvalLoggingHookloggers.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Tracked as EX-3854.
This was replied to by Claude.