From 6b02ea989ba4affc49a8ab204514ed65be66e3bf Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 11 Sep 2026 15:19:51 -0400 Subject: [PATCH 01/16] Drop APM traces when APM tracing is disabled and keep LLM Observability working Selecting a standalone sampler only when ASM/IAST/SCA is enabled left every other DD_APM_TRACING_ENABLED=false configuration on the default sampler, which keeps traces, so APM traces kept flowing. Drop them instead, as dd-trace-js and dd-trace-py do; LLM Observability spans are unaffected because they ship to the LLM Observability intake, which is independent of APM sampling priority. DD_TRACE_ENABLED=false installed the real LLM Observability SDK with no tracer to back it, and the no-op tracer's buildSpan() returns null, so every LLMObs.start*Span() call threw out of application code. Keep the SDK no-op. Co-Authored-By: Claude Opus 5 (1M context) --- .../datadog/trace/llmobs/LLMObsSystem.java | 8 +++ .../trace/llmobs/LLMObsSystemTest.java | 31 ++++++++++ .../apmtracingdisabled/Controller.java | 10 ++++ .../LlmObsApmTracingDisabledSmokeTest.groovy | 59 +++++++++++++++++++ .../LlmObsTraceDisabledSmokeTest.groovy | 38 ++++++++++++ .../trace/common/sampling/Sampler.java | 13 +++- .../trace/common/sampling/SamplerTest.java | 37 ++++++++++++ 7 files changed, 193 insertions(+), 3 deletions(-) create mode 100644 dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsSystemTest.java create mode 100644 dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmTracingDisabledSmokeTest.groovy create mode 100644 dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 864cf27eb2c..69f8e0bacb5 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -42,6 +42,14 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { return; } + // LLM Observability spans are backed by tracer spans, so without a tracer there is nothing to + // build them on: AgentTracer.get().buildSpan() returns null on the no-op tracer. Leave the + // no-op SDK implementations in place so the public API stays safe to call. + if (!config.isTraceEnabled()) { + LOGGER.debug("LLM Observability is disabled: tracing is disabled"); + return; + } + sco.createRemaining(config); String mlApp = config.getLlmObsMlApp(); diff --git a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsSystemTest.java b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsSystemTest.java new file mode 100644 index 00000000000..9abde374b2d --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsSystemTest.java @@ -0,0 +1,31 @@ +package datadog.trace.llmobs; + +import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED; +import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ENABLED; +import static org.junit.jupiter.api.Assertions.assertSame; + +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.noop.NoOpLLMObsSpan; +import datadog.trace.test.junit.utils.config.WithConfig; +import datadog.trace.test.util.DDJavaSpecification; +import org.junit.jupiter.api.Test; + +class LLMObsSystemTest extends DDJavaSpecification { + + /** + * LLM Observability spans are backed by tracer spans, so the subsystem must stay off when tracing + * is disabled. Starting it would install the real SDK implementation on top of the no-op tracer, + * whose {@code buildSpan()} returns null, and every span-starting call would throw. + */ + @WithConfig(key = LLMOBS_ENABLED, value = "true") + @WithConfig(key = TRACE_ENABLED, value = "false") + @Test + void staysNoOpWhenTracingIsDisabled() { + // The null SharedCommunicationObjects doubles as an assertion that start() returns before + // touching it: any use would throw. + LLMObsSystem.start(null, null); + + assertSame( + NoOpLLMObsSpan.INSTANCE, LLMObs.startLLMSpan("span", "model", "provider", null, null)); + } +} diff --git a/dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java b/dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java index d471fcb630b..6ca074d8e84 100644 --- a/dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java +++ b/dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java @@ -1,5 +1,7 @@ package datadog.smoketest.apmtracingdisabled; +import datadog.trace.api.llmobs.LLMObs; +import datadog.trace.api.llmobs.LLMObsSpan; import io.opentracing.Scope; import io.opentracing.Span; import io.opentracing.util.GlobalTracer; @@ -81,6 +83,14 @@ public void write( } } + @GetMapping("/llmobs") + public String llmObs() { + LLMObsSpan span = LLMObs.startLLMSpan("llm-call", "gpt-4", "openai", null, null); + span.annotateIO("input", "output"); + span.finish(); + return "llmobs"; + } + @GetMapping("/late-outbound") public String lateOutbound(@RequestParam(name = "url") String url) { final Span span = GlobalTracer.get().activeSpan(); diff --git a/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmTracingDisabledSmokeTest.groovy b/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmTracingDisabledSmokeTest.groovy new file mode 100644 index 00000000000..87cd018f051 --- /dev/null +++ b/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmTracingDisabledSmokeTest.groovy @@ -0,0 +1,59 @@ +package datadog.smoketest.apmtracingdisabled + +import datadog.trace.api.sampling.PrioritySampling +import okhttp3.Request +import spock.util.concurrent.PollingConditions + +/** + * With {@code DD_APM_TRACING_ENABLED=false}, LLM Observability keeps working — its spans go to the + * LLM Observability intake, which is independent of APM sampling — while the APM traces they ride + * on are dropped. + */ +class LlmObsApmTracingDisabledSmokeTest extends AbstractApmTracingDisabledSmokeTest { + + @Override + ProcessBuilder createProcessBuilder() { + final String[] processProperties = [ + "-Ddd.apm.tracing.enabled=false", + "-Ddd.llmobs.enabled=true", + "-Ddd.llmobs.ml.app=apm-tracing-disabled-smoketest", + "-Ddd.service.name=llmobs-apm-tracing-disabled-smoketest-app", + ] + return createProcess(processProperties) + } + + @Override + protected String traceAgentProtocolVersion() { + return '0.4' + } + + @Override + Closure decodedEvpProxyMessageCallback() { + return { String path, request -> + // The payload is msgpack; the path alone tells us the LLM Observability intake was reached. + return path.contains('api/v2/llmobs') ? path : null + } + } + + void 'LLMObs spans reach the LLM Observability intake while the APM trace is dropped'() { + setup: + final url = "http://localhost:${httpPort}/rest-api/llmobs" + final request = new Request.Builder().url(url).get().build() + + when: + final response = client.newCall(request).execute() + + then: + response.successful + waitForTraceCount(1) + + and: 'the APM trace is dropped' + checkRootSpanPrioritySampling(traces[0], PrioritySampling.SAMPLER_DROP) + hasApmDisabledTagOnEverySpan(traces[0]) + + and: 'the LLMObs span still reached the LLM Observability intake' + new PollingConditions(timeout: 30).eventually { + assert !evpProxyMessages.isEmpty() + } + } +} diff --git a/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy b/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy new file mode 100644 index 00000000000..5a44ebf0e81 --- /dev/null +++ b/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy @@ -0,0 +1,38 @@ +package datadog.smoketest.apmtracingdisabled + +import okhttp3.Request + +/** + * {@code DD_TRACE_ENABLED=false} leaves no tracer to back LLM Observability spans, so the SDK must + * stay no-op instead of throwing out of instrumented application code. + */ +class LlmObsTraceDisabledSmokeTest extends AbstractApmTracingDisabledSmokeTest { + + @Override + ProcessBuilder createProcessBuilder() { + final String[] processProperties = [ + "-Ddd.trace.enabled=false", + "-Ddd.llmobs.enabled=true", + "-Ddd.llmobs.ml.app=apm-tracing-disabled-smoketest", + "-Ddd.service.name=llmobs-trace-disabled-smoketest-app", + ] + return createProcess(processProperties) + } + + @Override + protected String traceAgentProtocolVersion() { + return '0.4' + } + + void 'the LLMObs SDK stays safe to call when tracing is disabled'() { + setup: + final url = "http://localhost:${httpPort}/rest-api/llmobs" + final request = new Request.Builder().url(url).get().build() + + when: + final response = client.newCall(request).execute() + + then: + response.successful + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java b/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java index 823f0d4959b..6f057782232 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java @@ -36,9 +36,16 @@ final class Builder { public static Sampler forConfig(final Config config, final TraceConfig traceConfig) { Sampler sampler; if (config != null) { - if (!config.isApmTracingEnabled() && isAsmEnabled(config)) { - log.debug("APM is disabled. Only 1 trace per minute will be sent."); - return new AsmStandaloneSampler(Clock.systemUTC()); + if (!config.isApmTracingEnabled()) { + if (isAsmEnabled(config)) { + log.debug( + "APM tracing is disabled, but ASM is enabled. Only 1 APM trace per minute will be sent."); + return new AsmStandaloneSampler(Clock.systemUTC()); + } + // No product needs a continuous APM trace, so drop them all. Products that keep their + // own traces (ASM, AI Guard) force-keep them, which this sampler cannot override. + log.debug("APM tracing is disabled. APM traces will be dropped."); + return new ForcePrioritySampler(PrioritySampling.SAMPLER_DROP, SamplingMechanism.DEFAULT); } final Map serviceRules = config.getTraceSamplingServiceRules(); final Map operationRules = config.getTraceSamplingOperationRules(); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java index e4658e7046b..98815701e7c 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java @@ -4,10 +4,12 @@ import static datadog.trace.api.config.AppSecConfig.APPSEC_SCA_ENABLED; import static datadog.trace.api.config.GeneralConfig.APM_TRACING_ENABLED; import static datadog.trace.api.config.IastConfig.IAST_ENABLED; +import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED; import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER; import static datadog.trace.api.config.TracerConfig.PRIORITY_SAMPLING; import static datadog.trace.api.config.TracerConfig.PRIORITY_SAMPLING_FORCE; import static datadog.trace.api.config.TracerConfig.TRACE_SAMPLE_RATE; +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -67,6 +69,23 @@ void asmStandaloneSamplerNotSelectedWhenApmTracingAndAsmNotEnabled() { assertFalse(sampler instanceof AsmStandaloneSampler); } + @WithConfig(key = APM_TRACING_ENABLED, value = "false") + @Test + void apmTracesDroppedWhenApmTracingDisabledAndNoOtherProductEnabled() { + assertApmTracesDropped(); + } + + /** + * LLM Observability rides the tracer but ships its spans to the LLM Observability intake, so + * disabling APM tracing must drop the APM traces without disabling the tracer. + */ + @WithConfig(key = APM_TRACING_ENABLED, value = "false") + @WithConfig(key = LLMOBS_ENABLED, value = "true") + @Test + void apmTracesDroppedWhenApmTracingDisabledAndLlmObsEnabled() { + assertApmTracesDropped(); + } + @Test void asmStandaloneSamplerNotSelectedWhenApmTracingEnabledAndAsmNotEnabled() { Config config = Config.get(); @@ -165,4 +184,22 @@ void spansBuiltWithOtlpEnabledAndPrioritySamplingDisabledHaveNonUnsetSamplingPri tracer.close(); } } + + private static void assertApmTracesDropped() { + Sampler sampler = Sampler.Builder.forConfig(Config.get(), null); + + assertInstanceOf(ForcePrioritySampler.class, sampler); + + CoreTracer tracer = CoreTracer.builder().writer(new ListWriter()).sampler(sampler).build(); + try { + DDSpan span = (DDSpan) tracer.buildSpan("datadog", "test").start(); + ((PrioritySampler) sampler).setSamplingPriority(span); + + assertEquals(SAMPLER_DROP, (int) span.getSamplingPriority()); + + span.finish(); + } finally { + tracer.close(); + } + } } From 5e30ea9b2dc0bb3cad0b128bb4e2a10686055327 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Mon, 14 Sep 2026 11:53:28 -0400 Subject: [PATCH 02/16] Clarify why the tracer-disabled and drop-all-traces branches exist Co-Authored-By: Claude Opus 5 --- .../src/main/java/datadog/trace/llmobs/LLMObsSystem.java | 9 +++++---- .../main/java/datadog/trace/common/sampling/Sampler.java | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 69f8e0bacb5..25b1cd6c1be 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -42,11 +42,12 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { return; } - // LLM Observability spans are backed by tracer spans, so without a tracer there is nothing to - // build them on: AgentTracer.get().buildSpan() returns null on the no-op tracer. Leave the - // no-op SDK implementations in place so the public API stays safe to call. + // dd.trace.enabled=false turns the whole tracer off, and LLMObs spans are backed by tracer + // spans: buildSpan() returns null on the no-op tracer, so starting up here would make every + // span-starting call throw into application code. Leave the no-op SDK installed instead. + // (dd.apm.tracing.enabled=false is different — the tracer stays, so LLMObs works.) if (!config.isTraceEnabled()) { - LOGGER.debug("LLM Observability is disabled: tracing is disabled"); + LOGGER.debug("LLM Observability is disabled: the tracer is disabled (dd.trace.enabled)"); return; } diff --git a/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java b/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java index 6f057782232..18f9667672a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java @@ -42,8 +42,9 @@ public static Sampler forConfig(final Config config, final TraceConfig traceConf "APM tracing is disabled, but ASM is enabled. Only 1 APM trace per minute will be sent."); return new AsmStandaloneSampler(Clock.systemUTC()); } - // No product needs a continuous APM trace, so drop them all. Products that keep their - // own traces (ASM, AI Guard) force-keep them, which this sampler cannot override. + // ASM is the only product needing a trickle of APM traces to stay in the service + // catalog, so with it off we can drop every APM trace. Traces a product does want + // (asm.keep, ai_guard.keep) are force-kept on the span, which no sampler can override. log.debug("APM tracing is disabled. APM traces will be dropped."); return new ForcePrioritySampler(PrioritySampling.SAMPLER_DROP, SamplingMechanism.DEFAULT); } From 592309351af4842a8d0c8a89f6597a36197f9f63 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Mon, 14 Sep 2026 13:13:21 -0400 Subject: [PATCH 03/16] Drop the LLMObs smoke tests and cover the behavior in SamplerTest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two new Groovy smoke tests would have needed the `tag: override groovy enforcement` label. What they proved end-to-end — a dropped APM trace is still handed to the writers, which is how LLM Observability spans still reach their intake — is now asserted in SamplerTest instead. Co-Authored-By: Claude Opus 5 --- .../apmtracingdisabled/Controller.java | 10 ---- .../LlmObsApmTracingDisabledSmokeTest.groovy | 59 ------------------- .../LlmObsTraceDisabledSmokeTest.groovy | 38 ------------ .../trace/common/sampling/SamplerTest.java | 17 +++++- 4 files changed, 16 insertions(+), 108 deletions(-) delete mode 100644 dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmTracingDisabledSmokeTest.groovy delete mode 100644 dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy diff --git a/dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java b/dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java index 6ca074d8e84..d471fcb630b 100644 --- a/dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java +++ b/dd-smoke-tests/apm-tracing-disabled/application/src/main/java/datadog/smoketest/apmtracingdisabled/Controller.java @@ -1,7 +1,5 @@ package datadog.smoketest.apmtracingdisabled; -import datadog.trace.api.llmobs.LLMObs; -import datadog.trace.api.llmobs.LLMObsSpan; import io.opentracing.Scope; import io.opentracing.Span; import io.opentracing.util.GlobalTracer; @@ -83,14 +81,6 @@ public void write( } } - @GetMapping("/llmobs") - public String llmObs() { - LLMObsSpan span = LLMObs.startLLMSpan("llm-call", "gpt-4", "openai", null, null); - span.annotateIO("input", "output"); - span.finish(); - return "llmobs"; - } - @GetMapping("/late-outbound") public String lateOutbound(@RequestParam(name = "url") String url) { final Span span = GlobalTracer.get().activeSpan(); diff --git a/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmTracingDisabledSmokeTest.groovy b/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmTracingDisabledSmokeTest.groovy deleted file mode 100644 index 87cd018f051..00000000000 --- a/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsApmTracingDisabledSmokeTest.groovy +++ /dev/null @@ -1,59 +0,0 @@ -package datadog.smoketest.apmtracingdisabled - -import datadog.trace.api.sampling.PrioritySampling -import okhttp3.Request -import spock.util.concurrent.PollingConditions - -/** - * With {@code DD_APM_TRACING_ENABLED=false}, LLM Observability keeps working — its spans go to the - * LLM Observability intake, which is independent of APM sampling — while the APM traces they ride - * on are dropped. - */ -class LlmObsApmTracingDisabledSmokeTest extends AbstractApmTracingDisabledSmokeTest { - - @Override - ProcessBuilder createProcessBuilder() { - final String[] processProperties = [ - "-Ddd.apm.tracing.enabled=false", - "-Ddd.llmobs.enabled=true", - "-Ddd.llmobs.ml.app=apm-tracing-disabled-smoketest", - "-Ddd.service.name=llmobs-apm-tracing-disabled-smoketest-app", - ] - return createProcess(processProperties) - } - - @Override - protected String traceAgentProtocolVersion() { - return '0.4' - } - - @Override - Closure decodedEvpProxyMessageCallback() { - return { String path, request -> - // The payload is msgpack; the path alone tells us the LLM Observability intake was reached. - return path.contains('api/v2/llmobs') ? path : null - } - } - - void 'LLMObs spans reach the LLM Observability intake while the APM trace is dropped'() { - setup: - final url = "http://localhost:${httpPort}/rest-api/llmobs" - final request = new Request.Builder().url(url).get().build() - - when: - final response = client.newCall(request).execute() - - then: - response.successful - waitForTraceCount(1) - - and: 'the APM trace is dropped' - checkRootSpanPrioritySampling(traces[0], PrioritySampling.SAMPLER_DROP) - hasApmDisabledTagOnEverySpan(traces[0]) - - and: 'the LLMObs span still reached the LLM Observability intake' - new PollingConditions(timeout: 30).eventually { - assert !evpProxyMessages.isEmpty() - } - } -} diff --git a/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy b/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy deleted file mode 100644 index 5a44ebf0e81..00000000000 --- a/dd-smoke-tests/apm-tracing-disabled/src/test/groovy/datadog/smoketest/apmtracingdisabled/LlmObsTraceDisabledSmokeTest.groovy +++ /dev/null @@ -1,38 +0,0 @@ -package datadog.smoketest.apmtracingdisabled - -import okhttp3.Request - -/** - * {@code DD_TRACE_ENABLED=false} leaves no tracer to back LLM Observability spans, so the SDK must - * stay no-op instead of throwing out of instrumented application code. - */ -class LlmObsTraceDisabledSmokeTest extends AbstractApmTracingDisabledSmokeTest { - - @Override - ProcessBuilder createProcessBuilder() { - final String[] processProperties = [ - "-Ddd.trace.enabled=false", - "-Ddd.llmobs.enabled=true", - "-Ddd.llmobs.ml.app=apm-tracing-disabled-smoketest", - "-Ddd.service.name=llmobs-trace-disabled-smoketest-app", - ] - return createProcess(processProperties) - } - - @Override - protected String traceAgentProtocolVersion() { - return '0.4' - } - - void 'the LLMObs SDK stays safe to call when tracing is disabled'() { - setup: - final url = "http://localhost:${httpPort}/rest-api/llmobs" - final request = new Request.Builder().url(url).get().build() - - when: - final response = client.newCall(request).execute() - - then: - response.successful - } -} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java index 98815701e7c..0720c0dcbc2 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java @@ -11,10 +11,12 @@ import static datadog.trace.api.config.TracerConfig.TRACE_SAMPLE_RATE; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; +import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.api.Config; import datadog.trace.common.writer.ListWriter; @@ -22,6 +24,7 @@ import datadog.trace.core.DDSpan; import datadog.trace.test.junit.utils.config.WithConfig; import datadog.trace.test.util.DDJavaSpecification; +import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.Test; class SamplerTest extends DDJavaSpecification { @@ -185,19 +188,31 @@ void spansBuiltWithOtlpEnabledAndPrioritySamplingDisabledHaveNonUnsetSamplingPri } } + /** + * Asserts the trace is marked dropped but still written. Products that ride the tracer and ship + * their spans elsewhere — LLM Observability sends them to its own intake — depend on the spans + * still reaching the writers, so dropping APM traces must mean a drop priority, not a discarded + * span. + */ private static void assertApmTracesDropped() { Sampler sampler = Sampler.Builder.forConfig(Config.get(), null); assertInstanceOf(ForcePrioritySampler.class, sampler); - CoreTracer tracer = CoreTracer.builder().writer(new ListWriter()).sampler(sampler).build(); + ListWriter writer = new ListWriter(); + CoreTracer tracer = CoreTracer.builder().writer(writer).sampler(sampler).build(); try { DDSpan span = (DDSpan) tracer.buildSpan("datadog", "test").start(); ((PrioritySampler) sampler).setSamplingPriority(span); assertEquals(SAMPLER_DROP, (int) span.getSamplingPriority()); + assertTrue(sampler.sample(span)); span.finish(); + writer.waitForTraces(1); + assertEquals(singletonList(span), writer.firstTrace()); + } catch (InterruptedException | TimeoutException e) { + throw new AssertionError("the dropped trace was never written", e); } finally { tracer.close(); } From 91793656e4a2d001ed00b09f1c137713fe3bbcc1 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Wed, 16 Sep 2026 16:01:35 -0400 Subject: [PATCH 04/16] Ignore the upstream sampling priority when APM tracing is disabled and no product marked the trace Co-Authored-By: Claude Opus 5 --- .../core/propagation/ContextInterpreter.java | 12 ++++++ .../propagation/DatadogHttpExtractorTest.java | 37 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ContextInterpreter.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ContextInterpreter.java index 14a6459ca0f..cb48a6b45ca 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ContextInterpreter.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ContextInterpreter.java @@ -19,6 +19,7 @@ import datadog.trace.api.DDSpanId; import datadog.trace.api.DDTraceId; import datadog.trace.api.Functions; +import datadog.trace.api.ProductTraceSource; import datadog.trace.api.TagMap; import datadog.trace.api.TraceConfig; import datadog.trace.api.TracePropagationStyle; @@ -72,6 +73,7 @@ public abstract class ContextInterpreter implements AgentPropagation.KeyClassifi private final boolean requestHeaderTagsCommaAllowed; private final int baggageMaxItems; private final int baggageMaxBytes; + private final boolean apmTracingEnabled; protected static final boolean LOG_EXTRACT_HEADER_NAMES = Config.get().isLogExtractHeaderNames(); private static final DDCache CACHE = DDCaches.newFixedSizeCache(64); @@ -89,6 +91,7 @@ protected ContextInterpreter(Config config) { this.requestHeaderTagsCommaAllowed = config.isRequestHeaderTagsCommaAllowed(); this.baggageMaxItems = config.getTraceBaggageMaxItems(); this.baggageMaxBytes = config.getTraceBaggageMaxBytes(); + this.apmTracingEnabled = config.isApmTracingEnabled(); } final TagMap.Ledger tagLedger() { @@ -345,6 +348,15 @@ private TagContext.HttpHeaders getHeaders() { } private int samplingPriorityOrDefault(DDTraceId traceId, int samplingPriority) { + if (!apmTracingEnabled + && (propagationTags == null + || propagationTags.getTraceSource() == ProductTraceSource.UNSET)) { + // With APM tracing disabled, an upstream decision is only meaningful when a product asked + // for the trace (_dd.p.ts). Otherwise it is an APM decision we must not inherit: the + // extracted priority is locked onto the span context at construction, so it would silence + // the local sampler before it ever votes. + return PrioritySampling.UNSET; + } return samplingPriority == PrioritySampling.UNSET || DDTraceId.ZERO.equals(traceId) ? defaultSamplingPriority() : samplingPriority; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/propagation/DatadogHttpExtractorTest.java b/dd-trace-core/src/test/java/datadog/trace/core/propagation/DatadogHttpExtractorTest.java index 5be84423457..eaf52aae393 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/propagation/DatadogHttpExtractorTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/propagation/DatadogHttpExtractorTest.java @@ -1,7 +1,9 @@ package datadog.trace.core.propagation; +import static datadog.trace.api.config.GeneralConfig.APM_TRACING_ENABLED; import static datadog.trace.api.config.TracerConfig.REQUEST_HEADER_TAGS_COMMA_ALLOWED; import static datadog.trace.api.config.TracerConfig.TRACE_BAGGAGE_MAX_ITEMS; +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; import static datadog.trace.api.sampling.PrioritySampling.UNSET; import static datadog.trace.bootstrap.instrumentation.api.ContextVisitors.stringValuesMap; import static datadog.trace.core.propagation.DatadogHttpCodec.DATADOG_TAGS_KEY; @@ -345,6 +347,41 @@ void extractMappedBaggageIsSubjectToTheItemLimit() { assertEquals(singletonMap(SOME_BAGGAGE, "mappedBaggageValue"), context.getBaggage()); } + /** + * With APM tracing disabled, an upstream sampling decision is only trustworthy when a product + * asked for the trace ({@code _dd.p.ts}). Otherwise it must not be inherited: the extracted + * priority is locked onto the span context at construction, which would silence the local + * sampler. + */ + @Nested + @WithConfig(key = APM_TRACING_ENABLED, value = "false") + class ApmTracingDisabled { + @Test + void upstreamPriorityDroppedWhenNoProductMarkedTheTrace() { + Map headers = new HashMap<>(); + headers.put(TRACE_ID_KEY, "1"); + headers.put(SPAN_ID_KEY, "2"); + headers.put(SAMPLING_PRIORITY_KEY, String.valueOf(SAMPLER_KEEP)); + + ExtractedContext context = (ExtractedContext) extractor.extract(headers, stringValuesMap()); + + assertEquals(UNSET, context.getSamplingPriority()); + } + + @Test + void upstreamPriorityKeptWhenProductMarkedTheTrace() { + Map headers = new HashMap<>(); + headers.put(TRACE_ID_KEY, "1"); + headers.put(SPAN_ID_KEY, "2"); + headers.put(SAMPLING_PRIORITY_KEY, String.valueOf(SAMPLER_KEEP)); + headers.put(DATADOG_TAGS_KEY, "_dd.p.ts=02"); + + ExtractedContext context = (ExtractedContext) extractor.extract(headers, stringValuesMap()); + + assertEquals(SAMPLER_KEEP, context.getSamplingPriority()); + } + } + @Nested class BaggageLimits extends AbstractOTBaggageTest { @Override From 80e580ee4c93892beeab420ed9e90d9fe2934a20 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Wed, 16 Sep 2026 16:21:15 -0400 Subject: [PATCH 05/16] Start LLM Observability when CI Visibility installs the tracer Co-Authored-By: Claude Opus 5 --- .../java/datadog/trace/llmobs/LLMObsSystem.java | 14 ++++++++------ .../datadog/trace/llmobs/LLMObsSystemTest.java | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 25b1cd6c1be..07174c0b8d7 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -42,12 +42,14 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { return; } - // dd.trace.enabled=false turns the whole tracer off, and LLMObs spans are backed by tracer - // spans: buildSpan() returns null on the no-op tracer, so starting up here would make every - // span-starting call throw into application code. Leave the no-op SDK installed instead. - // (dd.apm.tracing.enabled=false is different — the tracer stays, so LLMObs works.) - if (!config.isTraceEnabled()) { - LOGGER.debug("LLM Observability is disabled: the tracer is disabled (dd.trace.enabled)"); + // LLMObs spans are backed by tracer spans: buildSpan() returns null on the no-op tracer, so + // starting up with no CoreTracer installed would make every span-starting call throw into + // application code. Leave the no-op SDK installed instead. This mirrors the condition in + // TracerInstaller.installGlobalTracer — CI Visibility installs a CoreTracer even with + // dd.trace.enabled=false, and LLMObs works in that case. (dd.apm.tracing.enabled=false is + // different again — the tracer stays, so LLMObs works.) + if (!config.isTraceEnabled() && !config.isCiVisibilityEnabled()) { + LOGGER.debug("LLM Observability is disabled: no tracer is installed (dd.trace.enabled)"); return; } diff --git a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsSystemTest.java b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsSystemTest.java index 9abde374b2d..34f6050ca2f 100644 --- a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsSystemTest.java +++ b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsSystemTest.java @@ -1,8 +1,10 @@ package datadog.trace.llmobs; +import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_ENABLED; import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED; import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ENABLED; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import datadog.trace.api.llmobs.LLMObs; import datadog.trace.api.llmobs.noop.NoOpLLMObsSpan; @@ -28,4 +30,19 @@ void staysNoOpWhenTracingIsDisabled() { assertSame( NoOpLLMObsSpan.INSTANCE, LLMObs.startLLMSpan("span", "model", "provider", null, null)); } + + /** + * CI Visibility installs a {@code CoreTracer} even with tracing disabled (see {@code + * TracerInstaller#installGlobalTracer}), so the guard above must not fire in that combination — + * the tracer LLM Observability needs does exist. + */ + @WithConfig(key = LLMOBS_ENABLED, value = "true") + @WithConfig(key = TRACE_ENABLED, value = "false") + @WithConfig(key = CIVISIBILITY_ENABLED, value = "true") + @Test + void startsWhenCiVisibilityInstallsTheTracer() { + // Reaching the null SharedCommunicationObjects is the assertion: start() only dereferences it + // after the guard, so the NPE proves the subsystem was not short-circuited. + assertThrows(NullPointerException.class, () -> LLMObsSystem.start(null, null)); + } } From fe764e766b425de2878a76acda802df47d912ac9 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 18 Sep 2026 13:06:18 -0400 Subject: [PATCH 06/16] Dispatch the secondary queue before a flush completes Co-Authored-By: Claude Opus 5 --- .../common/writer/TraceProcessingWorker.java | 12 +++++ .../writer/TraceProcessingWorkerTest.java | 52 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/TraceProcessingWorker.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/TraceProcessingWorker.java index 9b70b09bc9e..479a20f3f69 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/TraceProcessingWorker.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/TraceProcessingWorker.java @@ -128,6 +128,11 @@ MessagePassingBlockingQueue getPrimaryQueue() { return primaryQueue; } + @VisibleForTesting + MessagePassingBlockingQueue getSecondaryQueue() { + return secondaryQueue; + } + private static MessagePassingBlockingQueue createQueue(int capacity) { return Queues.mpscBlockingConsumerArrayQueue(capacity); } @@ -195,6 +200,13 @@ public void onEvent(Object event) { // TODO populate `_sample_rate` metric in a way that accounts for lost/dropped traces payloadDispatcher.addTrace(trace); } else if (event instanceof FlushEvent) { + // Sweep the secondary queue first. The flush marker is only ever offered to the primary + // queue, so without this a trace published to the secondary queue can still be sitting + // there when the marker is serialized: flush() returns, close() interrupts this thread, + // and the trace is never dispatched. That is harmless for a sampled-out APM trace, but + // with APM tracing disabled the dropped trace also carries the LLM Observability spans, + // which have to reach their intake regardless of the APM sampling decision. + consumeFromSecondaryQueue(); payloadDispatcher.flush(); ((FlushEvent) event).sync(); } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java index 86dbfa61b22..98878865b95 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceProcessingWorkerTest.java @@ -23,6 +23,7 @@ import datadog.trace.test.util.DDJavaSpecification; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -148,6 +149,57 @@ void testAFlushShouldClearThePrimaryQueue() { } } + // ------------------------------------------------------------------------- + // Test 3b: a flush should dispatch the secondary queue before completing + // ------------------------------------------------------------------------- + + /** + * The flush marker is only offered to the primary queue, so a trace on the secondary queue has to + * be swept into the dispatcher before the flush completes — otherwise {@code close()} interrupts + * the serializer thread and the trace is never dispatched. Sampled-out APM traces can afford + * that, but with APM tracing disabled the dropped trace also carries the LLM Observability spans. + */ + @Test + void testAFlushShouldDispatchTheSecondaryQueueBeforeCompleting() { + List calls = Collections.synchronizedList(new ArrayList<>()); + PayloadDispatcherImpl dispatcher = mock(PayloadDispatcherImpl.class); + doAnswer( + inv -> { + calls.add("addTrace"); + return null; + }) + .when(dispatcher) + .addTrace(any()); + doAnswer( + inv -> { + calls.add("flush"); + return null; + }) + .when(dispatcher) + .flush(); + + try (TraceProcessingWorker worker = + new TraceProcessingWorker( + 10, + mock(HealthMetrics.class), + dispatcher, + () -> false, + FAST_LANE, + 100, + TimeUnit.SECONDS, // prevent heartbeats from helping the flush happen + null)) { + List trace = Collections.singletonList(mock(DDSpan.class)); + worker.getSecondaryQueue().offer(trace); + worker.start(); + boolean flushed = worker.flush(10, TimeUnit.SECONDS); + + assertTrue(flushed); + assertTrue(worker.getSecondaryQueue().isEmpty()); + // the trace reached the dispatcher, and it did so before the flush was reported complete + assertEquals(Arrays.asList("addTrace", "flush"), calls); + } + } + // ------------------------------------------------------------------------- // Test 4: should report failure if serialization fails // ------------------------------------------------------------------------- From a2a2a57de33286991f07097b67c1a003e233eca6 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 18 Sep 2026 14:13:10 -0400 Subject: [PATCH 07/16] Keep the standalone ASM trickle when AppSec activates at runtime Co-Authored-By: Claude Opus 5 --- .../sampling/ApmTracingDisabledSampler.java | 56 ++++++++++++ .../trace/common/sampling/Sampler.java | 12 ++- .../trace/common/sampling/SamplerTest.java | 85 ++++++++++++++++++- 3 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 dd-trace-core/src/main/java/datadog/trace/common/sampling/ApmTracingDisabledSampler.java diff --git a/dd-trace-core/src/main/java/datadog/trace/common/sampling/ApmTracingDisabledSampler.java b/dd-trace-core/src/main/java/datadog/trace/common/sampling/ApmTracingDisabledSampler.java new file mode 100644 index 00000000000..e99e8ac3070 --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/common/sampling/ApmTracingDisabledSampler.java @@ -0,0 +1,56 @@ +package datadog.trace.common.sampling; + +import datadog.trace.api.sampling.PrioritySampling; +import datadog.trace.api.sampling.SamplingMechanism; +import datadog.trace.bootstrap.ActiveSubsystems; +import datadog.trace.core.CoreSpan; +import java.time.Clock; + +/** + * Drops APM traces for {@code dd.apm.tracing.enabled=false}, unless AppSec is activated while the + * process is running, in which case it falls back to the standalone ASM trickle of 1 APM trace per + * minute. + * + *

The sampler is picked once, when the tracer is built, but AppSec can be activated later by + * Remote Configuration ({@code ASM_FEATURES}), which sets {@link ActiveSubsystems#APPSEC_ACTIVE} + * rather than changing the immutable {@link datadog.trace.api.Config} the choice was made from. The + * only place a sampler is rebuilt ({@code CoreTracer.ConfigSnapshot}) reacts to trace sampling rate + * and rule changes and rebuilds from the initial config, so it would not notice either. Deciding + * per trace instead keeps the service-catalog signal that standalone ASM needs once AppSec comes + * up. + * + *

This only widens when the trickle is sent: the startup case is handled before this sampler is + * ever built (see {@code Sampler.Builder.forConfig}), and AppSec is the only Application Security + * product that can be activated at runtime — IAST instrumentation is installed in {@code premain} + * and SCA is read once, so neither has a Remote Configuration product. + */ +public class ApmTracingDisabledSampler implements Sampler, PrioritySampler { + + private final AsmStandaloneSampler asmStandaloneSampler; + private final ForcePrioritySampler dropSampler; + + public ApmTracingDisabledSampler(final Clock clock) { + this.asmStandaloneSampler = new AsmStandaloneSampler(clock); + this.dropSampler = + new ForcePrioritySampler(PrioritySampling.SAMPLER_DROP, SamplingMechanism.DEFAULT); + } + + @Override + public > boolean sample(final T span) { + // Both delegates keep sending the trace to the agent so it can collect stats on dropped traces. + return isAsmActive() ? asmStandaloneSampler.sample(span) : dropSampler.sample(span); + } + + @Override + public > void setSamplingPriority(final T span) { + if (isAsmActive()) { + asmStandaloneSampler.setSamplingPriority(span); + } else { + dropSampler.setSamplingPriority(span); + } + } + + private static boolean isAsmActive() { + return ActiveSubsystems.APPSEC_ACTIVE; + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java b/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java index f7458f75d00..3cddb4d3c77 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java @@ -66,8 +66,11 @@ public static Sampler forConfig( // ASM is the only product needing a trickle of APM traces to stay in the service // catalog, so with it off we can drop every APM trace. Traces a product does want // (asm.keep, ai_guard.keep) are force-kept on the span, which no sampler can override. - log.debug("APM tracing is disabled. APM traces will be dropped."); - return new ForcePrioritySampler(PrioritySampling.SAMPLER_DROP, SamplingMechanism.DEFAULT); + // AppSec can still be activated later by Remote Configuration, so the drop is decided + // per trace rather than here. + log.debug( + "APM tracing is disabled. APM traces will be dropped, unless AppSec is activated at runtime."); + return new ApmTracingDisabledSampler(Clock.systemUTC()); } final Map serviceRules = config.getTraceSamplingServiceRules(); final Map operationRules = config.getTraceSamplingOperationRules(); @@ -141,6 +144,11 @@ public static Sampler forConfig( return sampler; } + /** + * Whether any Application Security product is on at startup. Only AppSec can change after this: + * IAST instrumentation is installed in {@code premain} and SCA is read once, so neither has a + * Remote Configuration product to activate it with. See {@link ApmTracingDisabledSampler}. + */ private static boolean isAsmEnabled(Config config) { return config.getAppSecActivation() == ProductActivation.FULLY_ENABLED || config.getIastActivation() == ProductActivation.FULLY_ENABLED diff --git a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java index 0720c0dcbc2..93c9f3d60d3 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java @@ -11,6 +11,7 @@ import static datadog.trace.api.config.TracerConfig.TRACE_SAMPLE_RATE; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; +import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -19,11 +20,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.api.Config; +import datadog.trace.bootstrap.ActiveSubsystems; import datadog.trace.common.writer.ListWriter; import datadog.trace.core.CoreTracer; import datadog.trace.core.DDSpan; import datadog.trace.test.junit.utils.config.WithConfig; import datadog.trace.test.util.DDJavaSpecification; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.Test; @@ -89,6 +93,40 @@ void apmTracesDroppedWhenApmTracingDisabledAndLlmObsEnabled() { assertApmTracesDropped(); } + /** + * AppSec can be activated by Remote Configuration after the sampler has been built, and that only + * flips {@link ActiveSubsystems#APPSEC_ACTIVE} — the immutable config the sampler was chosen from + * still says AppSec is off. Standalone ASM then needs its trickle of 1 APM trace per minute to + * keep the service in the service catalog, so the first trace has to be kept. + */ + @WithConfig(key = APM_TRACING_ENABLED, value = "false") + @Test + void apmTracesKeptOncePerMinuteWhenAppSecIsActivatedAtRuntime() { + Sampler sampler = Sampler.Builder.forConfig(Config.get(), null); + assertInstanceOf(ApmTracingDisabledSampler.class, sampler); + + // the trickle is one per minute, so only the first of the two traces is kept + withAppSecActive( + () -> + assertEquals( + asList((int) SAMPLER_KEEP, (int) SAMPLER_DROP), + samplingPrioritiesOfTraces(sampler, 2))); + } + + /** Deactivating AppSec at runtime puts the drop back. */ + @WithConfig(key = APM_TRACING_ENABLED, value = "false") + @Test + void apmTracesDroppedWhenAppSecIsDeactivatedAtRuntime() { + Sampler sampler = Sampler.Builder.forConfig(Config.get(), null); + + withAppSecActive( + () -> + assertEquals( + singletonList((int) SAMPLER_KEEP), samplingPrioritiesOfTraces(sampler, 1))); + + assertEquals(singletonList((int) SAMPLER_DROP), samplingPrioritiesOfTraces(sampler, 1)); + } + @Test void asmStandaloneSamplerNotSelectedWhenApmTracingEnabledAndAsmNotEnabled() { Config config = Config.get(); @@ -188,6 +226,51 @@ void spansBuiltWithOtlpEnabledAndPrioritySamplingDisabledHaveNonUnsetSamplingPri } } + /** + * Runs {@code assertions} with AppSec active, as Remote Configuration would leave it, always + * restoring the flag afterwards — it is global mutable state shared with every other test. + */ + private static void withAppSecActive(Runnable assertions) { + boolean wasActive = ActiveSubsystems.APPSEC_ACTIVE; + ActiveSubsystems.APPSEC_ACTIVE = true; + try { + assertions.run(); + } finally { + ActiveSubsystems.APPSEC_ACTIVE = wasActive; + } + } + + /** + * Runs {@code traceCount} traces through a tracer using {@code sampler} and returns the priority + * each one was written with. + * + *

The sampler is left to vote from the publish path ({@code + * TraceCollector.setSamplingPriorityIfNecessary}) rather than being called directly: with APM + * tracing disabled, {@code SamplingMechanism.APPSEC} is exempt from the sampling priority lock, + * so an extra direct call would both consume a slot of the one-per-minute trickle and overwrite + * the priority the publish path went on to assign. + */ + private static List samplingPrioritiesOfTraces(Sampler sampler, int traceCount) { + ListWriter writer = new ListWriter(); + CoreTracer tracer = CoreTracer.builder().writer(writer).sampler(sampler).build(); + try { + for (int i = 0; i < traceCount; i++) { + tracer.buildSpan("datadog", "test").start().finish(); + } + writer.waitForTraces(traceCount); + + List priorities = new ArrayList<>(); + for (List trace : writer) { + priorities.add(trace.get(0).getSamplingPriority()); + } + return priorities; + } catch (InterruptedException | TimeoutException e) { + throw new AssertionError("the traces were never written", e); + } finally { + tracer.close(); + } + } + /** * Asserts the trace is marked dropped but still written. Products that ride the tracer and ship * their spans elsewhere — LLM Observability sends them to its own intake — depend on the spans @@ -197,7 +280,7 @@ void spansBuiltWithOtlpEnabledAndPrioritySamplingDisabledHaveNonUnsetSamplingPri private static void assertApmTracesDropped() { Sampler sampler = Sampler.Builder.forConfig(Config.get(), null); - assertInstanceOf(ForcePrioritySampler.class, sampler); + assertInstanceOf(ApmTracingDisabledSampler.class, sampler); ListWriter writer = new ListWriter(); CoreTracer tracer = CoreTracer.builder().writer(writer).sampler(sampler).build(); From a153f2dacf2d812d201655a4c4bfe12619fa7bb0 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 25 Sep 2026 13:40:10 -0400 Subject: [PATCH 08/16] Let the APM traces drop override a manual.keep when APM tracing is disabled Co-Authored-By: Claude Opus 5 --- .../trace/common/sampling/SamplerTest.java | 62 +++++++++++++++++++ .../trace/api/sampling/SamplingMechanism.java | 16 ++++- .../api/sampling/SamplingMechanismTest.groovy | 2 + 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java index 93c9f3d60d3..1fb6d41b77e 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java @@ -11,6 +11,7 @@ import static datadog.trace.api.config.TracerConfig.TRACE_SAMPLE_RATE; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; +import static datadog.trace.api.sampling.PrioritySampling.USER_KEEP; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -20,7 +21,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.api.Config; +import datadog.trace.api.DDTags; +import datadog.trace.api.ProductTraceSource; import datadog.trace.bootstrap.ActiveSubsystems; +import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.common.writer.ListWriter; import datadog.trace.core.CoreTracer; import datadog.trace.core.DDSpan; @@ -29,6 +33,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; import org.junit.jupiter.api.Test; class SamplerTest extends DDJavaSpecification { @@ -93,6 +98,42 @@ void apmTracesDroppedWhenApmTracingDisabledAndLlmObsEnabled() { assertApmTracesDropped(); } + /** + * {@code manual.keep} force-keeps the span long before the sampler votes, locking the sampling + * priority. With APM tracing disabled the drop still has to win, or the trace would be indexed + * and billed as APM anyway — exactly what the setting opts out of. + */ + @WithConfig(key = APM_TRACING_ENABLED, value = "false") + @Test + void apmTracesDroppedWhenApmTracingDisabledAndTraceManuallyKept() { + Sampler sampler = Sampler.Builder.forConfig(Config.get(), null); + + assertEquals( + SAMPLER_DROP, + (int) samplingPriorityOfTrace(sampler, span -> span.setTag(DDTags.MANUAL_KEEP, true))); + } + + /** + * A trace a product asked for is not an APM trace the setting opts out of, so {@code manual.keep} + * still wins once {@code _dd.p.ts} is marked — here for ASM. This mirrors dd-trace-js, which + * honors {@code manual.keep} in standalone mode only alongside a product trace source. + */ + @WithConfig(key = APM_TRACING_ENABLED, value = "false") + @Test + void manuallyKeptTracesKeptWhenApmTracingDisabledAndMarkedForAsm() { + Sampler sampler = Sampler.Builder.forConfig(Config.get(), null); + + assertEquals( + USER_KEEP, + (int) + samplingPriorityOfTrace( + sampler, + span -> { + span.setTag(Tags.PROPAGATED_TRACE_SOURCE, ProductTraceSource.ASM); + span.setTag(DDTags.MANUAL_KEEP, true); + })); + } + /** * AppSec can be activated by Remote Configuration after the sampler has been built, and that only * flips {@link ActiveSubsystems#APPSEC_ACTIVE} — the immutable config the sampler was chosen from @@ -271,6 +312,27 @@ private static List samplingPrioritiesOfTraces(Sampler sampler, int tra } } + /** + * Runs a single trace through a tracer using {@code sampler}, applying {@code tagger} to its root + * span before it is finished, and returns the priority it was written with. + */ + private static int samplingPriorityOfTrace(Sampler sampler, Consumer tagger) { + ListWriter writer = new ListWriter(); + CoreTracer tracer = CoreTracer.builder().writer(writer).sampler(sampler).build(); + try { + DDSpan span = (DDSpan) tracer.buildSpan("datadog", "test").start(); + tagger.accept(span); + span.finish(); + writer.waitForTraces(1); + + return writer.firstTrace().get(0).getSamplingPriority(); + } catch (InterruptedException | TimeoutException e) { + throw new AssertionError("the trace was never written", e); + } finally { + tracer.close(); + } + } + /** * Asserts the trace is marked dropped but still written. Products that ride the tracer and ship * their spans elsewhere — LLM Observability sends them to its own intake — depend on the spans diff --git a/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java b/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java index 6b0cd397626..4f71b87f656 100644 --- a/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java +++ b/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java @@ -77,12 +77,26 @@ public static boolean validateWithSamplingPriority(int mechanism, int priority) /** * Returns true if sampling priority lock can be avoided for the given mechanism and priority * + *

With {@code dd.apm.tracing.enabled=false}, the drop decided for APM traces must also beat an + * already locked priority. Otherwise {@code manual.keep} — which force-keeps the span before the + * sampler ever votes — would leave the trace indexed and billed as APM, exactly what the setting + * opts out of. Traces a product does want are protected earlier: {@code + * TraceCollector#setSamplingPriorityIfNecessary} skips the sampler altogether when {@code + * _dd.p.ts} is marked for ASM or AI Guard, so only unmarked traces reach this drop. This matches + * dd-trace-js, whose standalone sampler honors {@code manual.keep} only alongside a product trace + * source tag. + * + *

The exemption is deliberately limited to a drop: a {@link #DEFAULT} keep must never override + * a priority someone else already set. + * * @param priority the sampling priority * @param mechanism the sampling mechanism * @return {@code true} if the sampling priority lock can be avoided, {@code false} otherwise */ public static boolean canAvoidSamplingPriorityLock(int priority, int mechanism) { - return (!Config.get().isApmTracingEnabled() && mechanism == SamplingMechanism.APPSEC) + return (!Config.get().isApmTracingEnabled() + && (mechanism == SamplingMechanism.APPSEC + || (mechanism == DEFAULT && priority == SAMPLER_DROP))) || (Config.get().isDataJobsEnabled() && mechanism == DATA_JOBS); } diff --git a/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy index 4a4890435c1..dd3268e65ad 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy @@ -108,6 +108,8 @@ class SamplingMechanismTest extends DDSpecification { APPSEC | UNSET | true APPSEC | SAMPLER_KEEP | true UNKNOWN | SAMPLER_KEEP | false + // the APM traces drop has to beat a manual.keep that already locked the priority + DEFAULT | SAMPLER_DROP | true DEFAULT | SAMPLER_KEEP | false AGENT_RATE | SAMPLER_KEEP | false REMOTE_AUTO_RATE | SAMPLER_KEEP | false From 5a9d467a1d857e0bcd8e6f7b11e2deb0b0badb8b Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 25 Sep 2026 14:43:27 -0400 Subject: [PATCH 09/16] Keep the APM traces drop from overriding a Data Jobs keep Co-Authored-By: Claude Opus 5 --- .../trace/common/sampling/SamplerTest.java | 19 +++++++++++++++++++ .../trace/api/sampling/SamplingMechanism.java | 9 ++++++++- .../api/sampling/SamplingMechanismTest.groovy | 1 - 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java index 1fb6d41b77e..a134df6d05c 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java @@ -3,6 +3,7 @@ import static datadog.trace.api.config.AppSecConfig.APPSEC_ENABLED; import static datadog.trace.api.config.AppSecConfig.APPSEC_SCA_ENABLED; import static datadog.trace.api.config.GeneralConfig.APM_TRACING_ENABLED; +import static datadog.trace.api.config.GeneralConfig.DATA_JOBS_ENABLED; import static datadog.trace.api.config.IastConfig.IAST_ENABLED; import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED; import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER; @@ -23,6 +24,7 @@ import datadog.trace.api.Config; import datadog.trace.api.DDTags; import datadog.trace.api.ProductTraceSource; +import datadog.trace.api.sampling.SamplingMechanism; import datadog.trace.bootstrap.ActiveSubsystems; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.common.writer.ListWriter; @@ -134,6 +136,23 @@ void manuallyKeptTracesKeptWhenApmTracingDisabledAndMarkedForAsm() { })); } + /** + * Data Jobs keeps its traces with a {@link SamplingMechanism#DATA_JOBS} priority rather than a + * {@code _dd.p.ts} mark, so the APM traces drop has to leave it alone. + */ + @WithConfig(key = APM_TRACING_ENABLED, value = "false") + @WithConfig(key = DATA_JOBS_ENABLED, value = "true") + @Test + void dataJobsTracesKeptWhenApmTracingDisabled() { + Sampler sampler = Sampler.Builder.forConfig(Config.get(), null); + + assertEquals( + USER_KEEP, + (int) + samplingPriorityOfTrace( + sampler, span -> span.setSamplingPriority(USER_KEEP, SamplingMechanism.DATA_JOBS))); + } + /** * AppSec can be activated by Remote Configuration after the sampler has been built, and that only * flips {@link ActiveSubsystems#APPSEC_ACTIVE} — the immutable config the sampler was chosen from diff --git a/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java b/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java index 4f71b87f656..4a5cf7920d0 100644 --- a/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java +++ b/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java @@ -89,6 +89,11 @@ public static boolean validateWithSamplingPriority(int mechanism, int priority) *

The exemption is deliberately limited to a drop: a {@link #DEFAULT} keep must never override * a priority someone else already set. * + *

Data Jobs is excluded because it has no {@code _dd.p.ts} bit to be protected by: it keeps + * its traces with a {@link #DATA_JOBS} priority instead, which the drop would overwrite. Marking + * {@link datadog.trace.api.ProductTraceSource#DJM} on the trace and skipping the sampler for it, + * the way ASM and AI Guard do, would let the drop stay unconditional here. + * * @param priority the sampling priority * @param mechanism the sampling mechanism * @return {@code true} if the sampling priority lock can be avoided, {@code false} otherwise @@ -96,7 +101,9 @@ public static boolean validateWithSamplingPriority(int mechanism, int priority) public static boolean canAvoidSamplingPriorityLock(int priority, int mechanism) { return (!Config.get().isApmTracingEnabled() && (mechanism == SamplingMechanism.APPSEC - || (mechanism == DEFAULT && priority == SAMPLER_DROP))) + || (mechanism == DEFAULT + && priority == SAMPLER_DROP + && !Config.get().isDataJobsEnabled()))) || (Config.get().isDataJobsEnabled() && mechanism == DATA_JOBS); } diff --git a/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy index dd3268e65ad..a19254800e6 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy @@ -108,7 +108,6 @@ class SamplingMechanismTest extends DDSpecification { APPSEC | UNSET | true APPSEC | SAMPLER_KEEP | true UNKNOWN | SAMPLER_KEEP | false - // the APM traces drop has to beat a manual.keep that already locked the priority DEFAULT | SAMPLER_DROP | true DEFAULT | SAMPLER_KEEP | false AGENT_RATE | SAMPLER_KEEP | false From 16e10829160febcc42e9ecb23753f5c91e38cc19 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 25 Sep 2026 14:50:58 -0400 Subject: [PATCH 10/16] Clarify that the CI Visibility carve-out only guarantees a live tracer Co-Authored-By: Claude Opus 5 --- .../src/main/java/datadog/trace/llmobs/LLMObsSystem.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 07174c0b8d7..892b55f15ca 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -44,10 +44,10 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { // LLMObs spans are backed by tracer spans: buildSpan() returns null on the no-op tracer, so // starting up with no CoreTracer installed would make every span-starting call throw into - // application code. Leave the no-op SDK installed instead. This mirrors the condition in - // TracerInstaller.installGlobalTracer — CI Visibility installs a CoreTracer even with - // dd.trace.enabled=false, and LLMObs works in that case. (dd.apm.tracing.enabled=false is - // different again — the tracer stays, so LLMObs works.) + // application code. Leave the no-op SDK installed instead. The condition mirrors + // TracerInstaller.installGlobalTracer, which still installs a CoreTracer for CI Visibility + // when dd.trace.enabled=false. Delivery is a separate question there: CI Visibility's trace + // sanitation drops any trace whose local root isn't a test span. if (!config.isTraceEnabled() && !config.isCiVisibilityEnabled()) { LOGGER.debug("LLM Observability is disabled: no tracer is installed (dd.trace.enabled)"); return; From 6a1ee87a40620d7d92e7ed8dd456996d35b51e98 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 9 Oct 2026 10:36:52 -0400 Subject: [PATCH 11/16] Test that sampled-out traces still deliver LLMObs spans Co-Authored-By: Claude Opus 5.5 --- .../trace/common/sampling/SamplerTest.java | 12 ---- .../writer/DDIntakeWriterCombinedTest.java | 70 +++++++++++++++++++ 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java index a134df6d05c..37dfeeb0142 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java @@ -5,7 +5,6 @@ import static datadog.trace.api.config.GeneralConfig.APM_TRACING_ENABLED; import static datadog.trace.api.config.GeneralConfig.DATA_JOBS_ENABLED; import static datadog.trace.api.config.IastConfig.IAST_ENABLED; -import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED; import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER; import static datadog.trace.api.config.TracerConfig.PRIORITY_SAMPLING; import static datadog.trace.api.config.TracerConfig.PRIORITY_SAMPLING_FORCE; @@ -89,17 +88,6 @@ void apmTracesDroppedWhenApmTracingDisabledAndNoOtherProductEnabled() { assertApmTracesDropped(); } - /** - * LLM Observability rides the tracer but ships its spans to the LLM Observability intake, so - * disabling APM tracing must drop the APM traces without disabling the tracer. - */ - @WithConfig(key = APM_TRACING_ENABLED, value = "false") - @WithConfig(key = LLMOBS_ENABLED, value = "true") - @Test - void apmTracesDroppedWhenApmTracingDisabledAndLlmObsEnabled() { - assertApmTracesDropped(); - } - /** * {@code manual.keep} force-keeps the span long before the sampler votes, locking the sampling * priority. With APM tracing disabled the drop still has to win, or the trace would be indexed diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterCombinedTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterCombinedTest.java index c53286cb106..bd63f037cec 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterCombinedTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/DDIntakeWriterCombinedTest.java @@ -1,6 +1,7 @@ package datadog.trace.common.writer; import static datadog.trace.common.writer.ddagent.Prioritization.ENSURE_TRACE; +import static datadog.trace.common.writer.ddagent.Prioritization.FAST_LANE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -16,6 +17,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import com.fasterxml.jackson.databind.ObjectMapper; import datadog.communication.http.OkHttpUtils; import datadog.communication.serialization.FlushingBuffer; import datadog.communication.serialization.msgpack.MsgPackWriter; @@ -24,6 +26,10 @@ import datadog.trace.agent.test.server.http.JavaTestHttpServer; import datadog.trace.api.civisibility.CiVisibilityWellKnownTags; import datadog.trace.api.intake.TrackType; +import datadog.trace.api.sampling.PrioritySampling; +import datadog.trace.api.sampling.SamplingMechanism; +import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.common.writer.ddintake.DDIntakeApi; import datadog.trace.common.writer.ddintake.DDIntakeMapperDiscovery; import datadog.trace.core.DDCoreJavaSpecification; @@ -31,20 +37,26 @@ import datadog.trace.core.monitor.HealthMetrics; import datadog.trace.core.monitor.TracerHealthMetrics; import datadog.trace.test.util.Flaky; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Phaser; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.msgpack.jackson.dataformat.MessagePackFactory; @Timeout(value = 10, unit = TimeUnit.SECONDS) class DDIntakeWriterCombinedTest extends DDCoreJavaSpecification { @@ -808,6 +820,64 @@ void statsdCommFailure() throws Exception { healthMetrics.close(); } + /** + * With APM tracing disabled every APM trace is sampled out, so the LLM Observability spans only + * reach their intake if the writer still serializes p0 traces. Built like the LLMObs writer in + * {@link WriterFactory}: no dropping policy and FAST_LANE prioritization. + */ + @SuppressWarnings("unchecked") + @Test + void sampledOutTraceStillDeliversLlmObsSpans() throws Exception { + DDIntakeApi api = mock(DDIntakeApi.class); + DDIntakeWriter writer = + DDIntakeWriter.builder() + .addTrack(TrackType.LLMOBS, api) + .prioritization(FAST_LANE) + .monitoring(monitoring) + .flushIntervalMilliseconds(-1) + .alwaysFlush(false) + .build(); + writer.start(); + clearInvocations(api); + + AtomicReference sent = new AtomicReference<>(); + doAnswer( + invocation -> { + sent.set(writeTo(invocation.getArgument(0))); + return RemoteApi.Response.success(200); + }) + .when(api) + .sendSerializedTraces(any()); + + DDSpan span = + (DDSpan) + dummyTracer + .buildSpan("datadog", "openai.request") + .withTag("_ml_obs_tag.span.kind", Tags.LLMOBS_LLM_SPAN_KIND) + .start(); + span.setSpanType(InternalSpanTypes.LLMOBS); + span.setSamplingPriority(PrioritySampling.SAMPLER_DROP, SamplingMechanism.DEFAULT); + + writer.write(Collections.singletonList(span)); + writer.flush(); + + verify(api, times(1)).sendSerializedTraces(argThat(payload -> payload.traceCount() == 1)); + Map result = + new ObjectMapper(new MessagePackFactory()).readValue(sent.get(), Map.class); + List> spans = (List>) result.get("spans"); + assertEquals(1, spans.size()); + assertEquals(String.valueOf(span.getSpanId()), spans.get(0).get("span_id")); + + writer.close(); + } + + private static byte[] writeTo(Payload payload) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + WritableByteChannel channel = Channels.newChannel(out); + payload.writeTo(channel); + return out.toByteArray(); + } + static String buildIntakePath(TrackType trackType, String apiVersion) { return String.format("/api/%s/%s", apiVersion, trackType.name().toLowerCase()); } From b1545561cd37f599b7e9b4f8802f94a114600801 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 9 Oct 2026 12:01:28 -0400 Subject: [PATCH 12/16] Match the extraction priority reset to TraceCollector's product check Co-Authored-By: Claude Opus 5.5 --- .../trace/core/propagation/ContextInterpreter.java | 12 +++++++----- .../core/propagation/DatadogHttpExtractorTest.java | 13 +++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ContextInterpreter.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ContextInterpreter.java index cb48a6b45ca..7e91658ccaf 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ContextInterpreter.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ContextInterpreter.java @@ -350,11 +350,13 @@ private TagContext.HttpHeaders getHeaders() { private int samplingPriorityOrDefault(DDTraceId traceId, int samplingPriority) { if (!apmTracingEnabled && (propagationTags == null - || propagationTags.getTraceSource() == ProductTraceSource.UNSET)) { - // With APM tracing disabled, an upstream decision is only meaningful when a product asked - // for the trace (_dd.p.ts). Otherwise it is an APM decision we must not inherit: the - // extracted priority is locked onto the span context at construction, so it would silence - // the local sampler before it ever votes. + || !ProductTraceSource.isProductMarked( + propagationTags.getTraceSource(), + ProductTraceSource.ASM, + ProductTraceSource.AI_GUARD))) { + // With APM tracing disabled, don't inherit an upstream APM decision: let the local sampler + // set the priority. Same products as TraceCollector.setSamplingPriorityIfNecessary. Needed + // when Data Jobs is enabled, where the drop can't override an already-set priority. return PrioritySampling.UNSET; } return samplingPriority == PrioritySampling.UNSET || DDTraceId.ZERO.equals(traceId) diff --git a/dd-trace-core/src/test/java/datadog/trace/core/propagation/DatadogHttpExtractorTest.java b/dd-trace-core/src/test/java/datadog/trace/core/propagation/DatadogHttpExtractorTest.java index eaf52aae393..f357b21888b 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/propagation/DatadogHttpExtractorTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/propagation/DatadogHttpExtractorTest.java @@ -380,6 +380,19 @@ void upstreamPriorityKeptWhenProductMarkedTheTrace() { assertEquals(SAMPLER_KEEP, context.getSamplingPriority()); } + + @Test + void upstreamPriorityDroppedWhenOnlyDsmMarkedTheTrace() { + Map headers = new HashMap<>(); + headers.put(TRACE_ID_KEY, "1"); + headers.put(SPAN_ID_KEY, "2"); + headers.put(SAMPLING_PRIORITY_KEY, String.valueOf(SAMPLER_KEEP)); + headers.put(DATADOG_TAGS_KEY, "_dd.p.ts=04"); + + ExtractedContext context = (ExtractedContext) extractor.extract(headers, stringValuesMap()); + + assertEquals(UNSET, context.getSamplingPriority()); + } } @Nested From ec10186d679846459c4477622d67f3869b1db2b9 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 9 Oct 2026 12:20:38 -0400 Subject: [PATCH 13/16] Shorten comments to the why Co-Authored-By: Claude Opus 5.5 --- .../datadog/trace/llmobs/LLMObsSystem.java | 9 +++------ .../sampling/ApmTracingDisabledSampler.java | 19 +++--------------- .../trace/common/sampling/Sampler.java | 13 +++--------- .../common/writer/TraceProcessingWorker.java | 8 ++------ .../trace/common/sampling/SamplerTest.java | 3 +-- .../propagation/DatadogHttpExtractorTest.java | 6 ++---- .../trace/api/sampling/SamplingMechanism.java | 20 ++++--------------- 7 files changed, 18 insertions(+), 60 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 892b55f15ca..b570c29b9d5 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -42,12 +42,9 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { return; } - // LLMObs spans are backed by tracer spans: buildSpan() returns null on the no-op tracer, so - // starting up with no CoreTracer installed would make every span-starting call throw into - // application code. Leave the no-op SDK installed instead. The condition mirrors - // TracerInstaller.installGlobalTracer, which still installs a CoreTracer for CI Visibility - // when dd.trace.enabled=false. Delivery is a separate question there: CI Visibility's trace - // sanitation drops any trace whose local root isn't a test span. + // LLMObs spans need a real tracer (buildSpan() returns null on the no-op one), so keep the + // no-op SDK when none is installed. Mirrors TracerInstaller.installGlobalTracer, which still + // installs one for CI Visibility. if (!config.isTraceEnabled() && !config.isCiVisibilityEnabled()) { LOGGER.debug("LLM Observability is disabled: no tracer is installed (dd.trace.enabled)"); return; diff --git a/dd-trace-core/src/main/java/datadog/trace/common/sampling/ApmTracingDisabledSampler.java b/dd-trace-core/src/main/java/datadog/trace/common/sampling/ApmTracingDisabledSampler.java index e99e8ac3070..907072259bb 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/sampling/ApmTracingDisabledSampler.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/sampling/ApmTracingDisabledSampler.java @@ -7,22 +7,9 @@ import java.time.Clock; /** - * Drops APM traces for {@code dd.apm.tracing.enabled=false}, unless AppSec is activated while the - * process is running, in which case it falls back to the standalone ASM trickle of 1 APM trace per - * minute. - * - *

The sampler is picked once, when the tracer is built, but AppSec can be activated later by - * Remote Configuration ({@code ASM_FEATURES}), which sets {@link ActiveSubsystems#APPSEC_ACTIVE} - * rather than changing the immutable {@link datadog.trace.api.Config} the choice was made from. The - * only place a sampler is rebuilt ({@code CoreTracer.ConfigSnapshot}) reacts to trace sampling rate - * and rule changes and rebuilds from the initial config, so it would not notice either. Deciding - * per trace instead keeps the service-catalog signal that standalone ASM needs once AppSec comes - * up. - * - *

This only widens when the trickle is sent: the startup case is handled before this sampler is - * ever built (see {@code Sampler.Builder.forConfig}), and AppSec is the only Application Security - * product that can be activated at runtime — IAST instrumentation is installed in {@code premain} - * and SCA is read once, so neither has a Remote Configuration product. + * Drops APM traces for {@code dd.apm.tracing.enabled=false}. If Remote Configuration activates + * AppSec at runtime, falls back to the standalone ASM trickle of 1 APM trace per minute, so it + * checks {@link ActiveSubsystems#APPSEC_ACTIVE} per trace. */ public class ApmTracingDisabledSampler implements Sampler, PrioritySampler { diff --git a/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java b/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java index 3cddb4d3c77..8e47348359c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/sampling/Sampler.java @@ -63,11 +63,8 @@ public static Sampler forConfig( "APM tracing is disabled, but ASM is enabled. Only 1 APM trace per minute will be sent."); return new AsmStandaloneSampler(Clock.systemUTC()); } - // ASM is the only product needing a trickle of APM traces to stay in the service - // catalog, so with it off we can drop every APM trace. Traces a product does want - // (asm.keep, ai_guard.keep) are force-kept on the span, which no sampler can override. - // AppSec can still be activated later by Remote Configuration, so the drop is decided - // per trace rather than here. + // Only ASM needs a trickle of APM traces, so drop them all unless AppSec is activated + // later by Remote Configuration. log.debug( "APM tracing is disabled. APM traces will be dropped, unless AppSec is activated at runtime."); return new ApmTracingDisabledSampler(Clock.systemUTC()); @@ -144,11 +141,7 @@ public static Sampler forConfig( return sampler; } - /** - * Whether any Application Security product is on at startup. Only AppSec can change after this: - * IAST instrumentation is installed in {@code premain} and SCA is read once, so neither has a - * Remote Configuration product to activate it with. See {@link ApmTracingDisabledSampler}. - */ + /** Whether any Application Security product is on at startup. */ private static boolean isAsmEnabled(Config config) { return config.getAppSecActivation() == ProductActivation.FULLY_ENABLED || config.getIastActivation() == ProductActivation.FULLY_ENABLED diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/TraceProcessingWorker.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/TraceProcessingWorker.java index 479a20f3f69..0fae63585e2 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/TraceProcessingWorker.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/TraceProcessingWorker.java @@ -200,12 +200,8 @@ public void onEvent(Object event) { // TODO populate `_sample_rate` metric in a way that accounts for lost/dropped traces payloadDispatcher.addTrace(trace); } else if (event instanceof FlushEvent) { - // Sweep the secondary queue first. The flush marker is only ever offered to the primary - // queue, so without this a trace published to the secondary queue can still be sitting - // there when the marker is serialized: flush() returns, close() interrupts this thread, - // and the trace is never dispatched. That is harmless for a sampled-out APM trace, but - // with APM tracing disabled the dropped trace also carries the LLM Observability spans, - // which have to reach their intake regardless of the APM sampling decision. + // The flush marker only goes to the primary queue, so drain the secondary queue too: + // sampled-out traces can carry LLM Observability spans that must still be sent. consumeFromSecondaryQueue(); payloadDispatcher.flush(); ((FlushEvent) event).sync(); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java index 37dfeeb0142..3f7c9d3739c 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/sampling/SamplerTest.java @@ -105,8 +105,7 @@ void apmTracesDroppedWhenApmTracingDisabledAndTraceManuallyKept() { /** * A trace a product asked for is not an APM trace the setting opts out of, so {@code manual.keep} - * still wins once {@code _dd.p.ts} is marked — here for ASM. This mirrors dd-trace-js, which - * honors {@code manual.keep} in standalone mode only alongside a product trace source. + * still wins once {@code _dd.p.ts} is marked — here for ASM. */ @WithConfig(key = APM_TRACING_ENABLED, value = "false") @Test diff --git a/dd-trace-core/src/test/java/datadog/trace/core/propagation/DatadogHttpExtractorTest.java b/dd-trace-core/src/test/java/datadog/trace/core/propagation/DatadogHttpExtractorTest.java index f357b21888b..bb3c1fc63f4 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/propagation/DatadogHttpExtractorTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/propagation/DatadogHttpExtractorTest.java @@ -348,10 +348,8 @@ void extractMappedBaggageIsSubjectToTheItemLimit() { } /** - * With APM tracing disabled, an upstream sampling decision is only trustworthy when a product - * asked for the trace ({@code _dd.p.ts}). Otherwise it must not be inherited: the extracted - * priority is locked onto the span context at construction, which would silence the local - * sampler. + * With APM tracing disabled, an upstream sampling decision is only kept when {@code _dd.p.ts} + * marks the trace for ASM or AI Guard. Otherwise the local sampler sets the priority. */ @Nested @WithConfig(key = APM_TRACING_ENABLED, value = "false") diff --git a/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java b/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java index 4a5cf7920d0..813125feccd 100644 --- a/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java +++ b/internal-api/src/main/java/datadog/trace/api/sampling/SamplingMechanism.java @@ -77,22 +77,10 @@ public static boolean validateWithSamplingPriority(int mechanism, int priority) /** * Returns true if sampling priority lock can be avoided for the given mechanism and priority * - *

With {@code dd.apm.tracing.enabled=false}, the drop decided for APM traces must also beat an - * already locked priority. Otherwise {@code manual.keep} — which force-keeps the span before the - * sampler ever votes — would leave the trace indexed and billed as APM, exactly what the setting - * opts out of. Traces a product does want are protected earlier: {@code - * TraceCollector#setSamplingPriorityIfNecessary} skips the sampler altogether when {@code - * _dd.p.ts} is marked for ASM or AI Guard, so only unmarked traces reach this drop. This matches - * dd-trace-js, whose standalone sampler honors {@code manual.keep} only alongside a product trace - * source tag. - * - *

The exemption is deliberately limited to a drop: a {@link #DEFAULT} keep must never override - * a priority someone else already set. - * - *

Data Jobs is excluded because it has no {@code _dd.p.ts} bit to be protected by: it keeps - * its traces with a {@link #DATA_JOBS} priority instead, which the drop would overwrite. Marking - * {@link datadog.trace.api.ProductTraceSource#DJM} on the trace and skipping the sampler for it, - * the way ASM and AI Guard do, would let the drop stay unconditional here. + *

With APM tracing disabled, the {@link #DEFAULT} drop overrides an already-set priority (e.g. + * {@code manual.keep}), so the trace isn't kept as APM. Traces marked for ASM or AI Guard never + * reach the sampler. The drop is not exempt when Data Jobs is enabled, so it can't overwrite a + * {@link #DATA_JOBS} keep. * * @param priority the sampling priority * @param mechanism the sampling mechanism From c0b43f15d95977781e72ad576a812acde948a060 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Fri, 9 Oct 2026 12:20:41 -0400 Subject: [PATCH 14/16] Test canAvoidSamplingPriorityLock with Data Jobs and APM tracing enabled Co-Authored-By: Claude Opus 5.5 --- .../api/sampling/SamplingMechanismTest.groovy | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy index a19254800e6..2952a31a12f 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/sampling/SamplingMechanismTest.groovy @@ -118,4 +118,29 @@ class SamplingMechanismTest extends DDSpecification { DATA_JOBS | SAMPLER_KEEP | false EXTERNAL_OVERRIDE | SAMPLER_KEEP | false } + + void 'Test canAvoidSamplingPriorityLock with Data Jobs enabled'(){ + setup: + injectSysConfig("dd.apm.tracing.enabled", "false") + injectSysConfig("dd.data.jobs.enabled", "true") + + expect: + canAvoidSamplingPriorityLock(priority, mechanism) == valid + + where: + mechanism | priority | valid + DEFAULT | SAMPLER_DROP | false + APPSEC | SAMPLER_KEEP | true + DATA_JOBS | USER_KEEP | true + } + + void 'Test canAvoidSamplingPriorityLock with APM tracing enabled'(){ + expect: + canAvoidSamplingPriorityLock(priority, mechanism) == valid + + where: + mechanism | priority | valid + DEFAULT | SAMPLER_DROP | false + APPSEC | SAMPLER_KEEP | false + } } From 56f612a391660da0770272b130c88ff3e3d05c21 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Tue, 13 Oct 2026 10:25:47 -0400 Subject: [PATCH 15/16] Keep only one standalone ASM trace per minute when traces share a millisecond Co-Authored-By: Claude Opus 5.5 --- .../trace/common/sampling/AsmStandaloneSampler.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/sampling/AsmStandaloneSampler.java b/dd-trace-core/src/main/java/datadog/trace/common/sampling/AsmStandaloneSampler.java index 54a2d750700..4772b5dc806 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/sampling/AsmStandaloneSampler.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/sampling/AsmStandaloneSampler.java @@ -51,8 +51,12 @@ public > void setSamplingPriority(final T span) { private boolean shouldSample() { long now = clock.millis(); - return lastSampleTime.updateAndGet( - lastTime -> now - lastTime >= RATE_IN_MILLISECONDS ? now : lastTime) - == now; + // Compare against the time before the update, not after: two traces landing in the same + // millisecond as the last sample would both see the updated value equal to now and both be + // kept, sending 2 APM traces a minute instead of 1. + long lastSample = + lastSampleTime.getAndUpdate( + lastTime -> now - lastTime >= RATE_IN_MILLISECONDS ? now : lastTime); + return now - lastSample >= RATE_IN_MILLISECONDS; } } From 2f3dea34b25ba8b351b2f86dcd776cd2bfdbfc57 Mon Sep 17 00:00:00 2001 From: "nicole.cybul" Date: Wed, 14 Oct 2026 14:31:04 -0400 Subject: [PATCH 16/16] Stub isApmTracingEnabled in W3COtelTraceStateContinuationTest mocks Co-Authored-By: Claude Opus 5.5 --- .../core/propagation/W3COtelTraceStateContinuationTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dd-trace-core/src/test/java/datadog/trace/core/propagation/W3COtelTraceStateContinuationTest.java b/dd-trace-core/src/test/java/datadog/trace/core/propagation/W3COtelTraceStateContinuationTest.java index 3d28facc067..336a121015b 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/propagation/W3COtelTraceStateContinuationTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/propagation/W3COtelTraceStateContinuationTest.java @@ -91,6 +91,7 @@ void compoundExtractionKeepsFirstPriorityAndReconcilesOtelState( when(config.getTracePropagationStylesToExtract()) .thenReturn(new LinkedHashSet<>(asList(DATADOG, TRACECONTEXT))); when(config.getxDatadogTagsMaxLength()).thenReturn(DEFAULT_TRACE_X_DATADOG_TAGS_MAX_LENGTH); + when(config.isApmTracingEnabled()).thenReturn(true); CoreTracer tracer = tracerBuilder().build(); try { @@ -134,6 +135,7 @@ void compoundExtractionWithoutDatadogPriorityReconcilesFirstLocalDecision() { when(config.getTracePropagationStylesToExtract()) .thenReturn(new LinkedHashSet<>(asList(DATADOG, TRACECONTEXT))); when(config.getxDatadogTagsMaxLength()).thenReturn(DEFAULT_TRACE_X_DATADOG_TAGS_MAX_LENGTH); + when(config.isApmTracingEnabled()).thenReturn(true); CoreTracer tracer = tracerBuilder().build(); try {