From f70fa560c722d394f2b5ff155e2e59b3b1e58f06 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 23 Oct 2026 14:56:03 -0400 Subject: [PATCH 1/3] Don't cache propagation tag values unique to a trace or span The tracestate's last parent id and the LLM Observability trace, parent and parent agent span ids went through the shared 128-entry tag value cache. They never repeat, so every lookup missed, allocated a cache element, and overwrote a slot holding a value that does repeat. They now skip the cache. Values that repeat, such as the decision maker, sampling rate and the trace id's high bits, still use it. Co-Authored-By: Claude Opus 5.5 --- .../propagation/ptags/DatadogPTagsCodec.java | 5 ++++- .../trace/core/propagation/ptags/PTagsCodec.java | 8 ++++++++ .../trace/core/propagation/ptags/TagValue.java | 16 ++++++++++++++++ .../core/propagation/ptags/W3CPTagsCodec.java | 8 ++++++-- 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java index 4d86f94fec6..55d7a53f5cc 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java @@ -102,7 +102,10 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value, int samp return tagsFactory.createInvalid(PROPAGATION_ERROR_DECODING_ERROR); } TagKey tagKey = TagKey.from(Encoding.DATADOG, value, tagPos, tagKeyEndsAt); - TagValue tagValue = TagValue.from(Encoding.DATADOG, value, tagValuePos, tagValueEndsAt); + TagValue tagValue = + hasUniqueValues(tagKey) + ? TagValue.uncached(Encoding.DATADOG, value, tagValuePos, tagValueEndsAt) + : TagValue.from(Encoding.DATADOG, value, tagValuePos, tagValueEndsAt); if (tagKey != null) { if (!tagKey.equals(UPSTREAM_SERVICES_DEPRECATED_TAG)) { if (!validateTagValue(tagKey, tagValue)) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java index 3dd2e248442..00f47ab951f 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java @@ -31,9 +31,17 @@ abstract class PTagsCodec { protected static final TagKey LLMOBS_PAGENT_SPAN_ID_TAG = TagKey.from("llmobs_pagent_span_id"); protected static final TagKey LLMOBS_PAGENT_NAME_TAG = TagKey.from("llmobs_pagent_name"); protected static final TagKey LLMOBS_PARENT_ID_TAG = TagKey.from("llmobs_parent_id"); + protected static final TagKey LLMOBS_SAMPLE_RATE_TAG = TagKey.from("llmobs_sr"); protected static final TagKey LLMOBS_SAMPLING_DECISION_TAG = TagKey.from("llmobs_sd"); + /** Whether values of {@code key} are unique to one trace or span, so not worth caching. */ + static boolean hasUniqueValues(TagKey key) { + return LLMOBS_TRACE_ID_TAG.equals(key) + || LLMOBS_PARENT_ID_TAG.equals(key) + || LLMOBS_PAGENT_SPAN_ID_TAG.equals(key); + } + /** * Encodes the header. {@code llmObsValues} are the LLM Observability values of the span being * injected; {@code null} writes the ones that arrived on the inbound headers instead, which is diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/TagValue.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/TagValue.java index a6c5f1bbf0b..6f70050b490 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/TagValue.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/TagValue.java @@ -35,6 +35,22 @@ static TagValue from(Encoding encoding, CharSequence s, int start, int end) { } } + /** + * Like {@link #from(Encoding, CharSequence, int, int)}, for a value unique to one trace or span, + * such as a span id. Caching those would never hit, and would evict values that do repeat. + */ + static TagValue uncached(Encoding encoding, CharSequence s, int start, int end) { + if (s == null || isValueInvalid(s, start, end)) { + if (log.isDebugEnabled()) { + log.debug("Invalid header h: {} s: {} b: {} e: {}", encoding, s, start, end); + } + return null; + } + return encoding == Encoding.W3C + ? produceW3C(s, hashW3C(s, start, end), start, end) + : produceDD(s, hashDD(s, start, end), start, end); + } + private static boolean isValueInvalid(CharSequence s, int start, int end) { return start < 0 || end <= 0 || s.length() < end; } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java index 49db92a3ab9..a1a7fb18142 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java @@ -194,11 +194,15 @@ private static R parse(String value, Result result) { } else if (keyLength == 1 && c == 'o') { origin = TagValue.from(Encoding.W3C, value, tagValuePos, tagValueEndsAt); } else if (keyLength == 1 && c == 'p') { - lastParentId = TagValue.from(Encoding.W3C, value, tagValuePos, tagValueEndsAt); + // a span id, so it never repeats + lastParentId = TagValue.uncached(Encoding.W3C, value, tagValuePos, tagValueEndsAt); } else { TagKey tagKey = TagKey.from(Encoding.W3C, value, tagPos, tagKeyEndsAt); if (tagKey != null) { - TagValue tagValue = TagValue.from(Encoding.W3C, value, tagValuePos, tagValueEndsAt); + TagValue tagValue = + hasUniqueValues(tagKey) + ? TagValue.uncached(Encoding.W3C, value, tagValuePos, tagValueEndsAt) + : TagValue.from(Encoding.W3C, value, tagValuePos, tagValueEndsAt); if (!tagKey.equals(UPSTREAM_SERVICES_DEPRECATED_TAG)) { if (!validateTagValue(tagKey, tagValue)) { log.warn( From ee992ef11f0285535980a57b6cd9cc10289e404b Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 23 Oct 2026 14:56:06 -0400 Subject: [PATCH 2/3] Test which propagation tag values are cached Co-Authored-By: Claude Opus 5.5 --- .../ptags/UniqueTagValueCachingTest.java | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 dd-trace-core/src/test/java/datadog/trace/core/propagation/ptags/UniqueTagValueCachingTest.java diff --git a/dd-trace-core/src/test/java/datadog/trace/core/propagation/ptags/UniqueTagValueCachingTest.java b/dd-trace-core/src/test/java/datadog/trace/core/propagation/ptags/UniqueTagValueCachingTest.java new file mode 100644 index 00000000000..6745961b865 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/core/propagation/ptags/UniqueTagValueCachingTest.java @@ -0,0 +1,57 @@ +package datadog.trace.core.propagation.ptags; + +import static datadog.trace.core.propagation.PropagationTags.HeaderType.DATADOG; +import static datadog.trace.core.propagation.PropagationTags.HeaderType.W3C; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; + +import datadog.trace.core.propagation.PropagationTags; +import org.junit.jupiter.api.Test; + +/** Values unique to a trace or span skip the tag value cache; values that repeat use it. */ +class UniqueTagValueCachingTest { + private static final PropagationTags.Factory FACTORY = PropagationTags.factory(); + + @Test + void lastParentIdIsNotCached() { + CharSequence first = + FACTORY.fromHeaderValue(W3C, "dd=s:1;p:00f067aa0ba902b7").getLastParentId(); + CharSequence second = + FACTORY.fromHeaderValue(W3C, "dd=s:1;p:00f067aa0ba902b7").getLastParentId(); + + assertEquals(first, second); + assertNotSame(first, second); + } + + @Test + void uncachedValueEqualsCachedValue() { + TagValue cached = TagValue.from(TagElement.Encoding.W3C, "x:abc", 2, 5); + TagValue uncached = TagValue.uncached(TagElement.Encoding.W3C, "x:abc", 2, 5); + + assertEquals(cached, uncached); + assertEquals(cached.hashCode(), uncached.hashCode()); + assertEquals( + cached.forType(TagElement.Encoding.DATADOG), uncached.forType(TagElement.Encoding.DATADOG)); + } + + @Test + void repeatedValuesAreStillCached() { + TagValue first = TagValue.from(TagElement.Encoding.DATADOG, "-4"); + TagValue second = TagValue.from(TagElement.Encoding.DATADOG, "-4"); + + assertSame(first, second); + } + + @Test + void llmObsIdsParseTheSameUncached() { + String header = "_dd.p.llmobs_trace_id=0123456789abcdef0123456789abcdef,_dd.p.dm=-1"; + + PropagationTags tags = FACTORY.fromHeaderValue(DATADOG, header); + + assertEquals( + "0123456789abcdef0123456789abcdef", + tags.createTagMap().get("_dd.p.llmobs_trace_id"), + tags.createTagMap().toString()); + } +} From 90533c36b3cf78fa8f6a5acdd774670da2a655da Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 23 Oct 2026 16:17:06 -0400 Subject: [PATCH 3/3] Choose cached or uncached tag values in one place Co-Authored-By: Claude Opus 5.5 --- .../propagation/ptags/DatadogPTagsCodec.java | 5 +---- .../core/propagation/ptags/PTagsCodec.java | 9 +++++++++ .../core/propagation/ptags/TagValue.java | 20 +++++++++++-------- .../core/propagation/ptags/W3CPTagsCodec.java | 5 +---- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java index 55d7a53f5cc..c0767b7bcfd 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/DatadogPTagsCodec.java @@ -102,10 +102,7 @@ PropagationTags fromHeaderValue(PTagsFactory tagsFactory, String value, int samp return tagsFactory.createInvalid(PROPAGATION_ERROR_DECODING_ERROR); } TagKey tagKey = TagKey.from(Encoding.DATADOG, value, tagPos, tagKeyEndsAt); - TagValue tagValue = - hasUniqueValues(tagKey) - ? TagValue.uncached(Encoding.DATADOG, value, tagValuePos, tagValueEndsAt) - : TagValue.from(Encoding.DATADOG, value, tagValuePos, tagValueEndsAt); + TagValue tagValue = tagValue(Encoding.DATADOG, tagKey, value, tagValuePos, tagValueEndsAt); if (tagKey != null) { if (!tagKey.equals(UPSTREAM_SERVICES_DEPRECATED_TAG)) { if (!validateTagValue(tagKey, tagValue)) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java index 00f47ab951f..29867d08d83 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/PTagsCodec.java @@ -35,6 +35,15 @@ abstract class PTagsCodec { protected static final TagKey LLMOBS_SAMPLE_RATE_TAG = TagKey.from("llmobs_sr"); protected static final TagKey LLMOBS_SAMPLING_DECISION_TAG = TagKey.from("llmobs_sd"); + /** + * The value of tag {@code key}, cached unless the key's values are unique to one trace or span. + */ + static TagValue tagValue(Encoding encoding, TagKey key, CharSequence s, int start, int end) { + return hasUniqueValues(key) + ? TagValue.uncached(encoding, s, start, end) + : TagValue.from(encoding, s, start, end); + } + /** Whether values of {@code key} are unique to one trace or span, so not worth caching. */ static boolean hasUniqueValues(TagKey key) { return LLMOBS_TRACE_ID_TAG.equals(key) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/TagValue.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/TagValue.java index 6f70050b490..d1610bc62d3 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/TagValue.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/TagValue.java @@ -20,10 +20,7 @@ static TagValue from(Encoding encoding, CharSequence s) { } static TagValue from(Encoding encoding, CharSequence s, int start, int end) { - if (s == null || isValueInvalid(s, start, end)) { - if (log.isDebugEnabled()) { - log.debug("Invalid header h: {} s: {} b: {} e: {}", encoding, s, start, end); - } + if (isInvalid(encoding, s, start, end)) { return null; } if (encoding == Encoding.W3C) { @@ -40,10 +37,7 @@ static TagValue from(Encoding encoding, CharSequence s, int start, int end) { * such as a span id. Caching those would never hit, and would evict values that do repeat. */ static TagValue uncached(Encoding encoding, CharSequence s, int start, int end) { - if (s == null || isValueInvalid(s, start, end)) { - if (log.isDebugEnabled()) { - log.debug("Invalid header h: {} s: {} b: {} e: {}", encoding, s, start, end); - } + if (isInvalid(encoding, s, start, end)) { return null; } return encoding == Encoding.W3C @@ -51,6 +45,16 @@ static TagValue uncached(Encoding encoding, CharSequence s, int start, int end) : produceDD(s, hashDD(s, start, end), start, end); } + private static boolean isInvalid(Encoding encoding, CharSequence s, int start, int end) { + if (s == null || isValueInvalid(s, start, end)) { + if (log.isDebugEnabled()) { + log.debug("Invalid header h: {} s: {} b: {} e: {}", encoding, s, start, end); + } + return true; + } + return false; + } + private static boolean isValueInvalid(CharSequence s, int start, int end) { return start < 0 || end <= 0 || s.length() < end; } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java index a1a7fb18142..9f292518b89 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ptags/W3CPTagsCodec.java @@ -199,10 +199,7 @@ private static R parse(String value, Result result) { } else { TagKey tagKey = TagKey.from(Encoding.W3C, value, tagPos, tagKeyEndsAt); if (tagKey != null) { - TagValue tagValue = - hasUniqueValues(tagKey) - ? TagValue.uncached(Encoding.W3C, value, tagValuePos, tagValueEndsAt) - : TagValue.from(Encoding.W3C, value, tagValuePos, tagValueEndsAt); + TagValue tagValue = tagValue(Encoding.W3C, tagKey, value, tagValuePos, tagValueEndsAt); if (!tagKey.equals(UPSTREAM_SERVICES_DEPRECATED_TAG)) { if (!validateTagValue(tagKey, tagValue)) { log.warn(