Skip to content

Share LLMObs parent resolution and adopt propagated context in OpenAI spans #12878

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ncybul wants to merge 3 commits into
base: master
Choose a base branch
from
+850 −181
Draft

Share LLMObs parent resolution and adopt propagated context in OpenAI spans #12878

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
96f45f6
Adopt propagated LLMObs context in openai.request spans
ncybul Oct 14, 2026
379c829
Pin LLMObs parent resolution edge cases before refactoring
ncybul Oct 14, 2026
e9ef593
Share LLMObs parent resolution between manual and OpenAI spans
ncybul Oct 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsContextPropagationSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
import datadog.trace.api.llmobs.LLMObsContext;
import datadog.trace.api.llmobs.LLMObsPropagationSource;
import datadog.trace.api.llmobs.LLMObsPropagationValues;
import datadog.trace.api.llmobs.LLMObsTraceId;
import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext;
import datadog.trace.llmobs.domain.LLMObsTraceId;
import javax.annotation.Nullable;

/**
Expand Down
153 changes: 43 additions & 110 deletions dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@
import datadog.trace.api.llmobs.GenAiApmTags;
import datadog.trace.api.llmobs.LLMObs;
import datadog.trace.api.llmobs.LLMObsContext;
import datadog.trace.api.llmobs.LLMObsPropagationValues;
import datadog.trace.api.llmobs.LLMObsParent;
import datadog.trace.api.llmobs.LLMObsSampler;
import datadog.trace.api.llmobs.LLMObsSpan;
import datadog.trace.api.llmobs.LLMObsTags;
import datadog.trace.api.llmobs.LLMObsTraceId;
import datadog.trace.api.telemetry.LLMObsMetricCollector;
import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext;
import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
import datadog.trace.bootstrap.instrumentation.api.Tags;
import java.util.ArrayList;
Expand Down Expand Up @@ -152,108 +152,40 @@ public DDLLMObsSpan(

span.setTag(SPAN_KIND, kind);
spanKind = kind;
// Resolve effective ml_app, parent_id, session_id, agent_version, agent attribution and
// sampling decision from the LLMObs context, all gated on trace-id consistency. A stale
// context from a different trace (e.g. async boundary leakage) must not contribute any of
// them. Every inherited value is read inside the one same-trace branch below, so a newly
// propagated tag cannot ship with a weaker gate of its own.
AgentSpanContext parent = LLMObsContext.current();
String parentSpanID = LLMObsContext.ROOT_SPAN_ID;
String resolvedTraceId = null;
String resolvedMlApp = mlApp;
String resolvedAgentVersion = agentVersion;
String sampleRate = null;
String samplingDecision = null;
String resolvedParentAgentSpanId = null;
String resolvedParentAgentName = null;
boolean inheritedInProcess = false;
if (null != parent) {
if (!parent.getTraceId().equals(span.getTraceId())) {
LOGGER.error(
"trace ID mismatch, retrieved parent from context trace_id={}, span_id={}, started span trace_id={}, span_id={}",
parent.getTraceId(),
parent.getSpanId(),
span.getTraceId(),
span.getSpanId());
} else {
inheritedInProcess = true;
parentSpanID = String.valueOf(parent.getSpanId());
// Stay on the enclosing span's LLMObs trace. Never re-seeded from the APM trace id here:
// the enclosing span may itself have adopted a trace id from another service, and every
// span below it has to keep reporting that one.
resolvedTraceId = LLMObsContext.currentTraceId();
// Inherit ml_app from the enclosing LLMObs span, if this span doesn't name its own, so a
// whole agent subtree stays in one application rather than each nested span falling back
// to the service default.
if (resolvedMlApp == null || resolvedMlApp.isEmpty()) {
String inherited = LLMObsContext.currentMlApp();
if (inherited != null && !inherited.isEmpty()) {
resolvedMlApp = inherited;
}
}
// Inherit session_id from parent context only when it belongs to the same trace.
// Matches dd-trace-py and dd-trace-js: session_id need only be set on the root
// span; descendants inherit transitively via context propagation.
if (sessionId == null || sessionId.isEmpty()) {
String inherited = LLMObsContext.currentSessionId();
if (inherited != null && !inherited.isEmpty()) {
sessionId = inherited;
}
}
// Inherit agent_version from the enclosing agent span, if this span doesn't set its own.
// An explicit value always wins, so a nested agent's own version overrides an ancestor's
// for its own subtree, matching session_id's explicit-wins semantics.
if (agentVersion == null || agentVersion.isEmpty()) {
String inherited = LLMObsContext.currentAgentVersion();
if (inherited != null && !inherited.isEmpty()) {
resolvedAgentVersion = inherited;
}
}
// Inherit the sampling decision from the context if present.
sampleRate = LLMObsContext.currentSampleRate();
samplingDecision = LLMObsContext.currentSamplingDecision();
// Inherit agent attribution: the nearest agent-kind ancestor. Overridden just below when
// this span is itself an agent.
resolvedParentAgentSpanId = LLMObsContext.currentParentAgentSpanId();
resolvedParentAgentName = LLMObsContext.currentParentAgentName();
}
}

// No usable in-process LLMObs parent, so fall back to what arrived from another service on
// the span context's propagation tags. Null when nothing did, which leaves every value below
// at the default it already holds.
LLMObsPropagationValues propagated =
inheritedInProcess ? null : span.spanContext().getExtractedLLMObsValues();
if (propagated != null) {
if (propagated.parentId != null) {
parentSpanID = propagated.parentId;
}
// Adopt the caller's LLMObs trace id so the trace stays whole across the boundary. The APM
// trace id can't stand in for it: an intermediate service that starts a fresh APM trace
// would split the LLMObs trace in two, and dd-trace-py has been propagating this since it
// gained the tag. The value arrives as decimal and is stored as hex.
resolvedTraceId = LLMObsTraceId.fromWire(propagated.traceId);
if (resolvedMlApp == null || resolvedMlApp.isEmpty()) {
resolvedMlApp = propagated.mlApp;
}
if (sessionId == null || sessionId.isEmpty()) {
sessionId = propagated.sessionId;
}
resolvedParentAgentSpanId = propagated.parentAgentSpanId;
if (resolvedParentAgentSpanId != null) {
resolvedParentAgentName = propagated.parentAgentName;
}
// Adopt the upstream sampling verdict so a distributed LLMObs trace is retained or dropped
// as a whole. Re-rolling locally only agrees with the caller when both services happen to
// be configured at the same rate; honouring the propagated decision removes that
// coincidence. Both tags are required together — the rate is meaningless without the
// decision it produced, and a lone decision would be reported against this service's rate,
// which is not the rate that produced it.
if (propagated.sampleRate != null && propagated.samplingDecision != null) {
sampleRate = propagated.sampleRate;
samplingDecision = propagated.samplingDecision;
}
}
// Resolve the LLMObs parent: the in-process context when it is on this trace, else what arrived
// from another service, else none. See LLMObsParent for the rules; what follows layers this
// span's own explicit values on top.
LLMObsParent parent = LLMObsParent.resolve(span);
if (parent.ignoredContext != null) {
LOGGER.error(
"trace ID mismatch, retrieved parent from context trace_id={}, span_id={}, started span trace_id={}, span_id={}",
parent.ignoredContext.getTraceId(),
parent.ignoredContext.getSpanId(),
span.getTraceId(),
span.getSpanId());
}
String parentSpanID = parent.parentId;
// Stay on the parent's LLMObs trace. Never re-seeded from the APM trace id when there is one:
// the parent may itself have adopted a trace id from another service, and every span below it
// has to keep reporting that one.
String resolvedTraceId = parent.traceId;
// An explicit value wins; an empty one counts as unset. Inheriting ml_app keeps a whole agent
// subtree in one application rather than each nested span falling back to the service
// default. session_id need only be set on the root span, matching dd-trace-py and dd-trace-js.
// A nested agent's own agent_version overrides an ancestor's for its own subtree.
String resolvedMlApp = isEmpty(mlApp) ? parent.mlApp : mlApp;
if (isEmpty(sessionId) && parent.sessionId != null) {
sessionId = parent.sessionId;
}
String resolvedAgentVersion =
isEmpty(agentVersion) && parent.agentVersion != null ? parent.agentVersion : agentVersion;
// Honouring an inherited verdict keeps a distributed LLMObs trace retained or dropped as a
// whole, rather than agreeing with the caller only when both are configured at the same rate.
String sampleRate = parent.sampleRate;
String samplingDecision = parent.samplingDecision;
// The nearest agent-kind ancestor. Overridden just below when this span is itself an agent.
String resolvedParentAgentSpanId = parent.parentAgentSpanId;
String resolvedParentAgentName = parent.parentAgentName;

// Root of an LLMObs trace: seed the LLMObs trace id from the APM one, in the shape the other
// tracers store it in — hex above 2^64, decimal below. The shape has to match what fromWire
Expand Down Expand Up @@ -286,14 +218,11 @@ public DDLLMObsSpan(
}

// Roll a decision only at the true root of an LLMObs trace
if (!inheritedInProcess && propagated == null) {
if (parent.isRoot()) {
sampleRate = sampler.formattedRate();
samplingDecision =
sampler.sample(span.getTraceId().toLong())
? LLMObsContext.SAMPLING_DECISION_SAMPLED
: LLMObsContext.SAMPLING_DECISION_DROPPED;
samplingDecision = sampler.decide(span.getTraceId());
}
// Both tags move together for the same reason they are adopted together above: a rate without
// Both tags move together for the same reason LLMObsParent adopts them together: a rate without
// the decision it produced says nothing, and a decision on its own would be read against a
// rate that did not produce it.
if (sampleRate != null && samplingDecision != null) {
Expand Down Expand Up @@ -838,4 +767,8 @@ public String getLLMObsTraceId() {
public long getSpanId() {
return span.getSpanId();
}

private static boolean isEmpty(String value) {
return value == null || value.isEmpty();
}
}
70 changes: 70 additions & 0 deletions ...t/agent-llmobs/src/test/java/datadog/trace/llmobs/LLMObsContextPropagationSourceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,76 @@ void continuationDoesNotRerollWhenTheUpstreamSentNoVerdict() {
}
}

/**
* An agent span on the receiving side is its own descendants' nearest agent: the upstream
* attribution it received is replaced, for itself and for everything below it, by its own id and
* name.
*/
@Test
void agentSpanReceivingPropagatedAttributionBecomesTheParentAgent() {
Map<String, String> inbound = producerCarrier("checkout", "sess-42");
try (ContextScope consumeScope = startLocalChildScope(extractSpan(inbound))) {
DDLLMObsSpan workerAgent = newSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "worker-agent", null, null);
try {
assertEquals(
String.valueOf(workerAgent.getSpanId()), LLMObsContext.currentParentAgentSpanId());
assertEquals("worker-agent", LLMObsContext.currentParentAgentName());
assertEquals("sess-42", LLMObsContext.currentSessionId());
assertEquals("checkout", LLMObsContext.currentMlApp());
} finally {
workerAgent.finish();
}
}
}

/**
* A queue consumer whose previous message left an LLMObs scope open on the thread. The leaked
* context is on another trace, so it is ignored and the span takes what arrived with the current
* message rather than starting a new LLMObs trace.
*/
@Test
void staleContextFromAnotherTraceDoesNotShadowThePropagatedOne() {
Map<String, String> inbound = producerCarrier("checkout", "sess-42");

AgentSpan staleApm = AgentTracer.get().buildSpan("apm", "previous.message").start();
try (ContextScope stale = LLMObsContext.attach(staleApm.spanContext(), "stale-session")) {
try (ContextScope consumeScope = startLocalChildScope(extractSpan(inbound))) {
DDLLMObsSpan worker = newSpan(Tags.LLMOBS_TOOL_SPAN_KIND, "handler", null, null);
try {
assertEquals("sess-42", LLMObsContext.currentSessionId());
assertEquals("checkout", LLMObsContext.currentMlApp());
assertEquals("dispatcher", LLMObsContext.currentParentAgentName());
} finally {
worker.finish();
}
}
} finally {
staleApm.finish();
}
}

/** An explicit empty session id counts as unset, so the propagated one still applies. */
@Test
void emptyExplicitSessionIdInheritsThePropagatedOne() {
Map<String, String> inbound = producerCarrier("checkout", "sess-42");

try (ContextScope consumeScope = startLocalChildScope(extractSpan(inbound))) {
DDLLMObsSpan worker = newSpan(Tags.LLMOBS_TOOL_SPAN_KIND, "handler", null, "");
try {
assertEquals("sess-42", LLMObsContext.currentSessionId());
// And an in-process child naming an empty session inherits the worker's.
DDLLMObsSpan child = newSpan(Tags.LLMOBS_LLM_SPAN_KIND, "generate", null, "");
try {
assertEquals("sess-42", LLMObsContext.currentSessionId());
} finally {
child.finish();
}
} finally {
worker.finish();
}
}
}

@Test
void peerSpanDoesNotInheritAFinishedSpansInjectedContext() {
try (ContextScope apmScope = startRootApmScope()) {
Expand Down
15 changes: 15 additions & 0 deletions .../agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanAgentVersionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,21 @@ void noVersionSetAnywhereMeansNoTagOnAnySpanInTheSubtree() {
}
}

@Test
void emptyExplicitVersionCountsAsUnsetAndInheritsFromParent() {
DDLLMObsSpan agent = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "agent1", "v3");
try (ContextScope ignored = AgentTracer.activateSpan(spanOf(agent))) {
DDLLMObsSpan child = llmObsSpan(Tags.LLMOBS_TOOL_SPAN_KIND, "tool1", "");
try {
assertEquals("v3", spanOf(child).getTag(AGENT_VERSION_TAG));
} finally {
child.finish();
}
} finally {
agent.finish();
}
}

@Test
void childDoesNotInheritAgentVersionWhenStaleContextIsFromADifferentTrace()
throws ReflectiveOperationException {
Expand Down
Loading
Loading