Resolve Feature Flags enablement from instrumenter config

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -135,7 +135,8 @@ private enum AgentFeature {
     AGENTLESS_LOG_SUBMISSION(GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED, false),
     APP_LOGS_COLLECTION(GeneralConfig.APP_LOGS_COLLECTION_ENABLED, false),
     LLMOBS(LlmObsConfig.LLMOBS_ENABLED, false),
-    LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false);
+    LLMOBS_AGENTLESS(LlmObsConfig.LLMOBS_AGENTLESS_ENABLED, false),
+    FEATURE_FLAGS(FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED, true);
 
     private final String configKey;
     private final String systemProp;
@@ -284,7 +285,7 @@ public static void start(
     agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION);
     appLogsCollectionEnabled = isFeatureEnabled(AgentFeature.APP_LOGS_COLLECTION);
     llmObsEnabled = isFeatureEnabled(AgentFeature.LLMOBS);
-    featureFlaggingEnabled = isFeatureFlaggingEnabled();
+    featureFlaggingEnabled = isFeatureEnabled(AgentFeature.FEATURE_FLAGS);
 
     // setup writers when llmobs is enabled to accomodate apm and llmobs
     if (llmObsEnabled) {
@@ -1796,45 +1797,6 @@ private static boolean isFeatureEnabled(AgentFeature feature) {
     }
   }
 
-  private static boolean isFeatureFlaggingEnabled() {
-    final Boolean providerEnabled =
-        featureFlaggingBooleanSetting(FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED);
-    final String configurationSource =
-        featureFlaggingSetting(FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE);
-    final Boolean legacyProviderEnabled =
-        featureFlaggingBooleanSetting(FeatureFlaggingConfig.EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED);
-
-    return FeatureFlaggingConfig.resolveConfiguration(
-            providerEnabled, configurationSource, legacyProviderEnabled)
-        .isEnabled();
-  }
-
-  @SuppressFBWarnings(
-      value = "NP_BOOLEAN_RETURN_NULL",
-      justification = "A null value preserves the distinction between absent and explicitly false")
-  private static Boolean featureFlaggingBooleanSetting(final String configKey) {
-    final String value = featureFlaggingSetting(configKey);
-    if (value == null) {
-      return null;
-    }
-    return Boolean.parseBoolean(value) || "1".equals(value);
-  }
-
-  private static String featureFlaggingSetting(final String configKey) {
-    final String systemProperty = propertyNameToSystemPropertyName(configKey);
-    String value = SystemProperties.get(systemProperty);
-    if (value == null) {
-      value = getStableConfig(FLEET, configKey);
-    }
-    if (value == null) {
-      value = ddGetEnv(systemProperty);
-    }
-    if (value == null) {
-      value = getStableConfig(LOCAL, configKey);
-    }
-    return value;
-  }
-
   /**
    * @see datadog.trace.api.ProductActivation#fromString(String)
    */

dd-java-agent/agent-installer/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java

@@ -347,6 +347,9 @@ public static Set<InstrumenterModule.TargetSystem> getEnabledSystems() {
     if (cfg.isLlmObsEnabled()) {
       enabledSystems.add(InstrumenterModule.TargetSystem.LLMOBS);
     }
+    if (cfg.isFeatureFlaggingEnabled()) {
+      enabledSystems.add(InstrumenterModule.TargetSystem.FEATURE_FLAGS);
+    }
     return enabledSystems;
   }
 

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java

@@ -44,6 +44,7 @@ public abstract class InstrumenterModule implements Instrumenter {
    *   <li>{@link TargetSystem#USM usm}
    *   <li>{@link TargetSystem#CONTEXT_TRACKING context-tracking}
    *   <li>{@link TargetSystem#RASP rasp}
+   *   <li>{@link TargetSystem#FEATURE_FLAGS feature-flags}
    * </ul>
    */
   public enum TargetSystem {
@@ -56,6 +57,7 @@ public enum TargetSystem {
     LLMOBS,
     CONTEXT_TRACKING,
     RASP,
+    FEATURE_FLAGS,
   }
 
   private static final Logger log = LoggerFactory.getLogger(InstrumenterModule.class);

dd-java-agent/instrumentation/graal/graal-native-image-20.0/src/main/java/datadog/trace/instrumentation/graal/nativeimage/NativeImageGeneratorRunnerInstrumentation.java

@@ -85,6 +85,7 @@ public static void onEnter(@Advice.Argument(value = 0, readOnly = false) String[
               + "datadog.trace.api.config.OtlpConfig$Protocol:build_time,"
               + "datadog.trace.api.config.OtlpConfig$Temporality:build_time,"
               + "datadog.trace.api.EventTracker:build_time,"
+              + "datadog.trace.api.FeatureFlaggingSource:build_time,"
               + "datadog.trace.api.InstrumenterConfig:build_time,"
               + "datadog.trace.api.Functions:build_time,"
               + "datadog.trace.api.Functions$NewInstance:build_time,"

internal-api/src/main/java/datadog/trace/api/Config.java

@@ -750,8 +750,6 @@
 import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS;
 import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS;
 import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED;
-import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.isSupportedConfigurationSource;
-import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.resolveConfiguration;
 import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY;
 import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.MULTI_WRITER_TYPE;
 import static datadog.trace.bootstrap.instrumentation.api.WriterConstants.OTLP_WRITER_TYPE;
@@ -769,7 +767,6 @@
 import datadog.trace.api.config.OtlpConfig;
 import datadog.trace.api.config.ProfilingConfig;
 import datadog.trace.api.config.TracerConfig;
-import datadog.trace.api.featureflag.config.FeatureFlaggingConfig;
 import datadog.trace.api.iast.IastContext;
 import datadog.trace.api.iast.IastDetectionMode;
 import datadog.trace.api.iast.telemetry.Verbosity;
@@ -1258,8 +1255,6 @@ public static String getHostName() {
 
   private final int remoteConfigMaxExtraServices;
 
-  private final boolean featureFlaggingProviderEnabled;
-  private final String featureFlaggingConfigurationSource;
   private final String featureFlaggingConfigurationSourceAgentlessBaseUrl;
   private final int featureFlaggingConfigurationSourcePollIntervalSeconds;
   private final int featureFlaggingConfigurationSourceRequestTimeoutSeconds;
@@ -2951,33 +2946,19 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
         configProvider.getInteger(
             REMOTE_CONFIG_MAX_EXTRA_SERVICES, DEFAULT_REMOTE_CONFIG_MAX_EXTRA_SERVICES);
 
-    final Boolean configuredFeatureFlaggingProviderEnabled =
-        configProvider.getBoolean(FEATURE_FLAGS_ENABLED);
-    final Boolean legacyFeatureFlaggingProviderEnabled =
-        configProvider.getBoolean(EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED);
-    final String configuredFeatureFlaggingConfigurationSource =
-        configProvider.isSet(FEATURE_FLAGS_CONFIGURATION_SOURCE)
-            ? configProvider.getString(FEATURE_FLAGS_CONFIGURATION_SOURCE)
-            : null;
-    final FeatureFlaggingConfig.Resolution resolvedFeatureFlaggingConfiguration =
-        resolveConfiguration(
-            configuredFeatureFlaggingProviderEnabled,
-            configuredFeatureFlaggingConfigurationSource,
-            legacyFeatureFlaggingProviderEnabled);
-    if (legacyFeatureFlaggingProviderEnabled != null) {
+    if (configProvider.getBoolean(EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED) != null) {
       log.warn(
           "Setting {} is deprecated. Use {} and {} instead.",
           EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED,
           FEATURE_FLAGS_ENABLED,
           FEATURE_FLAGS_CONFIGURATION_SOURCE);
     }
-    if (!isSupportedConfigurationSource(configuredFeatureFlaggingConfigurationSource)) {
+    if (instrumenterConfig.getFeatureFlaggingConfigurationSource()
+        == FeatureFlaggingSource.INVALID) {
       log.warn(
           "Unsupported Feature Flagging configuration source; provider disabled: '{}'",
-          resolvedFeatureFlaggingConfiguration.getSource());
+          configProvider.getString(FEATURE_FLAGS_CONFIGURATION_SOURCE));
     }
-    featureFlaggingProviderEnabled = resolvedFeatureFlaggingConfiguration.isEnabled();
-    featureFlaggingConfigurationSource = resolvedFeatureFlaggingConfiguration.getSource();
     featureFlaggingConfigurationSourceAgentlessBaseUrl =
         configProvider.getStringNotEmpty(
             FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, null);
@@ -4940,11 +4921,12 @@ public int getRemoteConfigMaxExtraServices() {
   }
 
   public boolean isFeatureFlaggingProviderEnabled() {
-    return featureFlaggingProviderEnabled;
+    return instrumenterConfig.isFeatureFlaggingEnabled();
   }
 
   public String getFeatureFlaggingConfigurationSource() {
-    return featureFlaggingConfigurationSource;
+    final FeatureFlaggingSource source = instrumenterConfig.getFeatureFlaggingConfigurationSource();
+    return source == FeatureFlaggingSource.UNSET ? null : source.name().toLowerCase(Locale.ROOT);
   }
 
   public String getFeatureFlaggingConfigurationSourceAgentlessBaseUrl() {
@@ -6916,9 +6898,9 @@ public String toString() {
         + ", remoteConfigIntegrityCheckEnabled="
         + remoteConfigIntegrityCheckEnabled
         + ", featureFlaggingProviderEnabled="
-        + featureFlaggingProviderEnabled
+        + isFeatureFlaggingProviderEnabled()
         + ", featureFlaggingConfigurationSource="
-        + featureFlaggingConfigurationSource
+        + getFeatureFlaggingConfigurationSource()
         + ", featureFlaggingConfigurationSourceAgentlessBaseUrl="
         + featureFlaggingConfigurationSourceAgentlessBaseUrl
         + ", featureFlaggingConfigurationSourcePollIntervalSeconds="

internal-api/src/main/java/datadog/trace/api/FeatureFlaggingSource.java

@@ -0,0 +1,13 @@
+package datadog.trace.api;
+
+/** The Feature Flags configuration source. */
+public enum FeatureFlaggingSource {
+  /** No source is configured. */
+  UNSET,
+  /** The configured source is not supported. */
+  INVALID,
+  /** The flag configuration is polled from the Datadog CDN. */
+  AGENTLESS,
+  /** The flag configuration is received through Remote Configuration. */
+  REMOTE_CONFIG
+}

internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java

@@ -29,6 +29,10 @@
 import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_OTEL_ENABLED;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_USM_ENABLED;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_WEBSOCKET_MESSAGES_ENABLED;
+import static datadog.trace.api.FeatureFlaggingSource.AGENTLESS;
+import static datadog.trace.api.FeatureFlaggingSource.INVALID;
+import static datadog.trace.api.FeatureFlaggingSource.REMOTE_CONFIG;
+import static datadog.trace.api.FeatureFlaggingSource.UNSET;
 import static datadog.trace.api.config.AppSecConfig.API_SECURITY_ENDPOINT_COLLECTION_ENABLED;
 import static datadog.trace.api.config.AppSecConfig.APPSEC_ENABLED;
 import static datadog.trace.api.config.AppSecConfig.APPSEC_RASP_ENABLED;
@@ -98,8 +102,12 @@
 import static datadog.trace.api.config.TraceInstrumentationConfig.UNSAFE_CLASS_INJECTION;
 import static datadog.trace.api.config.TraceInstrumentationConfig.VISITOR_CLASS_PARSING;
 import static datadog.trace.api.config.UsmConfig.USM_ENABLED;
+import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED;
+import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE;
+import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED;
 import static datadog.trace.util.CollectionUtils.tryMakeImmutableList;
 import static datadog.trace.util.CollectionUtils.tryMakeImmutableSet;
+import static java.lang.Boolean.FALSE;
 import static java.util.Collections.singletonList;
 
 import datadog.environment.JavaVirtualMachine;
@@ -175,6 +183,8 @@ public class InstrumenterConfig {
   private final boolean usmEnabled;
   private final boolean telemetryEnabled;
   private final boolean llmObsEnabled;
+  private final boolean featureFlaggingEnabled;
+  private final FeatureFlaggingSource featureFlaggingConfigurationSource;
 
   private final String traceExtensionsPath;
 
@@ -276,6 +286,21 @@ private InstrumenterConfig() {
             CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT,
             DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT);
     traceEnabled = configProvider.getBoolean(TRACE_ENABLED, DEFAULT_TRACE_ENABLED);
+    // The product switch wins, then an explicit source, then the deprecated experimental toggle.
+    final Boolean featureFlagsEnabled = configProvider.getBoolean(FEATURE_FLAGS_ENABLED);
+    final FeatureFlaggingSource featureFlagsSource = featureFlaggingSource(configProvider);
+    final Boolean legacyFeatureFlagsEnabled =
+        configProvider.getBoolean(EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED);
+    if (FALSE.equals(featureFlagsEnabled) || featureFlagsSource != UNSET) {
+      featureFlaggingEnabled = !FALSE.equals(featureFlagsEnabled) && featureFlagsSource != INVALID;
+      featureFlaggingConfigurationSource = featureFlagsSource;
+    } else if (legacyFeatureFlagsEnabled != null) {
+      featureFlaggingEnabled = legacyFeatureFlagsEnabled;
+      featureFlaggingConfigurationSource = legacyFeatureFlagsEnabled ? REMOTE_CONFIG : UNSET;
+    } else {
+      featureFlaggingEnabled = true;
+      featureFlaggingConfigurationSource = AGENTLESS;
+    }
     traceOtelEnabled = configProvider.getBoolean(TRACE_OTEL_ENABLED, DEFAULT_TRACE_OTEL_ENABLED);
     metricsOtelEnabled =
         configProvider.getBoolean(METRICS_OTEL_ENABLED, DEFAULT_METRICS_OTEL_ENABLED);
@@ -491,6 +516,30 @@ public boolean isTraceEnabled() {
     return traceEnabled;
   }
 
+  public boolean isFeatureFlaggingEnabled() {
+    return featureFlaggingEnabled;
+  }
+
+  /**
+   * @return the Feature Flags configuration source, {@link FeatureFlaggingSource#UNSET} when none
+   *     is configured or {@link FeatureFlaggingSource#INVALID} when unsupported.
+   */
+  public FeatureFlaggingSource getFeatureFlaggingConfigurationSource() {
+    return featureFlaggingConfigurationSource;
+  }
+
+  private static FeatureFlaggingSource featureFlaggingSource(final ConfigProvider configProvider) {
+    final String value = configProvider.getString(FEATURE_FLAGS_CONFIGURATION_SOURCE);
+    if (value == null || value.trim().isEmpty()) {
+      return UNSET;
+    }
+    final FeatureFlaggingSource source =
+        configProvider.getEnum(
+            FEATURE_FLAGS_CONFIGURATION_SOURCE, FeatureFlaggingSource.class, INVALID);
+    // UNSET is not a valid value to configure.
+    return source == UNSET ? INVALID : source;
+  }
+
   public boolean isTraceOtelEnabled() {
     return traceOtelEnabled;
   }

internal-api/src/test/java/datadog/trace/api/ConfigTest.java

@@ -3667,16 +3667,16 @@ void agentlessFeatureFlagTimingUsesPositiveConfiguredValues() throws Exception {
   }
 
   @TableTest({
-    "scenario          | value             | expected         ",
-    "not set           |                   | agentless        ",
-    "empty             | ''                | agentless        ",
-    "blank             | '   '             | agentless        ",
-    "mixed case padded | ' ReMoTe_ConFiG ' | remote_config    ",
-    "unknown source    | not-a-real-source | not-a-real-source",
-    "padded uppercase  | ' OFFLINE '       | offline          "
+    "scenario             | value             | expected     ",
+    "not set              |                   | agentless    ",
+    "empty                | ''                | agentless    ",
+    "blank                | '   '             | agentless    ",
+    "case insensitive     | ReMoTe_ConFiG     | remote_config",
+    "padded               | ' remote_config ' | invalid      ",
+    "unknown source       | not-a-real-source | invalid      ",
+    "unset is not a value | unset             | invalid      "
   })
-  void featureFlagConfigurationSourceNormalizesValue(String value, String expected)
-      throws Exception {
+  void featureFlagConfigurationSourceParsesValue(String value, String expected) throws Exception {
     Properties properties = new Properties();
     if (value != null) {
       properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE, value);
@@ -3697,8 +3697,8 @@ void featureFlagConfigurationSourceNormalizesValue(String value, String expected
     "source wins over legacy disabled   |                 | remote_config | false                 | true            | remote_config ",
     "provider disabled wins over legacy | false           | agentless     | true                  | false           | agentless     ",
     "legacy disabled wins over provider | true            |               | false                 | false           |               ",
-    "unknown source                     |                 | not-a-source  |                       | false           | not-a-source  ",
-    "offline source                     |                 | offline       | true                  | false           | offline       "
+    "unknown source                     |                 | not-a-source  |                       | false           | invalid       ",
+    "offline source                     |                 | offline       | true                  | false           | invalid       "
   })
   void featureFlagConfigurationAppliesMigrationPrecedence(
       Boolean providerEnabled,

internal-api/src/test/java/datadog/trace/api/InstrumenterConfigFeatureFlaggingTest.java

@@ -0,0 +1,51 @@
+package datadog.trace.api;
+
+import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED;
+import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE;
+import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_ENABLED;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import datadog.trace.bootstrap.config.provider.ConfigProvider;
+import java.util.Properties;
+import org.tabletest.junit.TableTest;
+
+class InstrumenterConfigFeatureFlaggingTest {
+
+  @TableTest({
+    "scenario                  | enabled | source          | legacy  | resolvedEnabled | resolvedSource",
+    "defaults                  |         |                 |         | true            | AGENTLESS     ",
+    "blank source              |         | ' '             |         | true            | AGENTLESS     ",
+    "legacy enabled            |         |                 | 'true'  | true            | REMOTE_CONFIG ",
+    "legacy disabled           |         |                 | 'false' | false           | UNSET         ",
+    "source wins over legacy   |         | 'agentless'     | 'true'  | true            | AGENTLESS     ",
+    "case insensitive source   |         | 'Remote_Config' | 'false' | true            | REMOTE_CONFIG ",
+    "disabled wins over source | 'false' | 'agentless'     | 'true'  | false           | AGENTLESS     ",
+    "disabled without source   | 'false' |                 |         | false           | UNSET         ",
+    "unsupported source        |         | 'offline'       | 'true'  | false           | INVALID       ",
+    "padded source             |         | ' agentless '   |         | false           | INVALID       ",
+    "unset is not a value      |         | 'unset'         |         | false           | INVALID       "
+  })
+  void resolvesFeatureFlagging(
+      final String enabled,
+      final String source,
+      final String legacy,
+      final boolean resolvedEnabled,
+      final FeatureFlaggingSource resolvedSource) {
+    final Properties properties = new Properties();
+    if (enabled != null) {
+      properties.setProperty(FEATURE_FLAGS_ENABLED, enabled);
+    }
+    if (source != null) {
+      properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE, source);
+    }
+    if (legacy != null) {
+      properties.setProperty(EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED, legacy);
+    }
+
+    final InstrumenterConfig config =
+        new InstrumenterConfig(ConfigProvider.withPropertiesOverride(properties));
+
+    assertEquals(resolvedEnabled, config.isFeatureFlaggingEnabled());
+    assertEquals(resolvedSource, config.getFeatureFlaggingConfigurationSource());
+  }
+}