From 6680c2bb187d0bdd36ef992e518805da73194f39 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 29 Jul 2026 19:00:24 -0400 Subject: [PATCH 01/11] Fix JSON span event encoding in v1 payloads --- .../common/writer/ddagent/TraceMapperV1.java | 23 ++++++++- .../trace/core/DDSpanSerializationTest.java | 48 +++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java index 819ff2021be..944a99c2992 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java @@ -1,6 +1,7 @@ package datadog.trace.common.writer.ddagent; import static datadog.communication.http.OkHttpUtils.msgpackRequestBodyOf; +import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; @@ -11,6 +12,7 @@ import datadog.communication.serialization.Writable; import datadog.communication.serialization.msgpack.MsgPackWriter; import datadog.environment.JavaVirtualMachine; +import datadog.json.JsonReader; import datadog.trace.api.Config; import datadog.trace.api.DDTags; import datadog.trace.api.DDTraceId; @@ -202,12 +204,12 @@ private void encodeSpanLinks( private void encodeSpanEvents(Writable writable, int fieldId, Object eventsObject) { writable.writeInt(fieldId); - if (!(eventsObject instanceof List) || ((List) eventsObject).isEmpty()) { + List events = parseSpanEvents(eventsObject); + if (events.isEmpty()) { writable.startArray(0); return; } - List events = (List) eventsObject; int encodableCount = 0; for (Object event : events) { if (isEncodableSpanEvent(event)) { @@ -236,6 +238,23 @@ private void encodeSpanEvents(Writable writable, int fieldId, Object eventsObjec } } + private List parseSpanEvents(Object eventsObject) { + if (eventsObject instanceof List) { + return (List) eventsObject; + } + if (eventsObject instanceof CharSequence) { + try (JsonReader reader = new JsonReader(eventsObject.toString())) { + Object events = reader.nextValue(); + if (events instanceof List) { + return (List) events; + } + } catch (IOException e) { + log.debug("Failed to parse span events from JSON", e); + } + } + return emptyList(); + } + private boolean isEncodableSpanEvent(Object event) { if (!(event instanceof Map)) { return false; diff --git a/dd-trace-core/src/test/java/datadog/trace/core/DDSpanSerializationTest.java b/dd-trace-core/src/test/java/datadog/trace/core/DDSpanSerializationTest.java index aecf3517072..5ec74bc7748 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/DDSpanSerializationTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/DDSpanSerializationTest.java @@ -1,5 +1,6 @@ package datadog.trace.core; +import static datadog.trace.api.DDTags.SPAN_EVENTS; import static datadog.trace.api.DDTags.SPAN_LINKS; import static datadog.trace.api.config.GeneralConfig.EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED; import static datadog.trace.api.config.TracerConfig.TRACE_BAGGAGE_TAG_KEYS; @@ -33,6 +34,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.Channels; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -393,6 +395,52 @@ void serializeTraceWithSpanLinksAsStructuredLinksOnlyV1() throws Exception { tracer.close(); } + @Test + void serializeTraceWithJsonSpanEventsV1() throws Exception { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + DDSpanContext context = createSpanContext(tracer, Collections.emptyMap(), null, true, true, 1); + context.setTag( + SPAN_EVENTS, + "[{\"time_unix_nano\":1234567890,\"name\":\"event.one\"," + + "\"attributes\":{\"str\":\"value\",\"int\":42,\"double\":12.5," + + "\"bool\":true,\"arr\":[\"x\",7,2.5,false]}}," + + "{\"time_unix_nano\":1234567891,\"name\":\"event.two\"}]"); + DDSpan span = DDSpan.create("test", 0, context, null); + + V1PayloadReader.V1Span payload = V1PayloadReader.readFirstSpan(serializeV1Payload(span)); + + assertFalse(payload.getAttributes().containsKey(SPAN_EVENTS)); + assertEquals(2, payload.getEvents().size()); + V1PayloadReader.V1SpanEvent firstEvent = payload.getEvents().get(0); + assertEquals(1234567890L, firstEvent.getTimeUnixNano()); + assertEquals("event.one", firstEvent.getName()); + assertEquals("value", firstEvent.getAttributes().get("str")); + assertEquals(42L, firstEvent.getAttributes().get("int")); + assertEquals(12.5d, firstEvent.getAttributes().get("double")); + assertEquals(true, firstEvent.getAttributes().get("bool")); + assertEquals(Arrays.asList("x", 7L, 2.5d, false), firstEvent.getAttributes().get("arr")); + + V1PayloadReader.V1SpanEvent secondEvent = payload.getEvents().get(1); + assertEquals(1234567891L, secondEvent.getTimeUnixNano()); + assertEquals("event.two", secondEvent.getName()); + assertEquals(Collections.emptyMap(), secondEvent.getAttributes()); + tracer.close(); + } + + @Test + void serializeTraceWithMalformedJsonSpanEventsAsEmptyV1() throws Exception { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + DDSpanContext context = createSpanContext(tracer, Collections.emptyMap(), null, true, true, 1); + context.setTag(SPAN_EVENTS, "[{"); + DDSpan span = DDSpan.create("test", 0, context, null); + + V1PayloadReader.V1Span payload = V1PayloadReader.readFirstSpan(serializeV1Payload(span)); + + assertFalse(payload.getAttributes().containsKey(SPAN_EVENTS)); + assertEquals(Collections.emptyList(), payload.getEvents()); + tracer.close(); + } + @Test void serializeTraceWithFlatMapTagV04() throws Exception { CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); From ffe6112ff0a3a0e32c5ea1e41b2ca04cd8269ad9 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 29 Jul 2026 19:06:03 -0400 Subject: [PATCH 02/11] Enable `v1` by default to run the tests on CI. --- .../communication/ddagent/DDAgentFeaturesDiscovery.java | 4 ++-- .../ddagent/DDAgentFeaturesDiscoveryTest.groovy | 7 ++++--- .../src/main/java/datadog/trace/api/ConfigDefaults.java | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java index 16be2e84b98..398435a2589 100644 --- a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java +++ b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java @@ -4,7 +4,7 @@ import static datadog.communication.http.OkHttpUtils.msgpackRequestBodyOf; import static datadog.communication.http.OkHttpUtils.prepareRequest; import static datadog.communication.serialization.msgpack.MsgPackWriter.FIXARRAY; -import static datadog.trace.api.ProtocolVersion.V0_4; +import static datadog.trace.api.ProtocolVersion.V1_0; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; @@ -118,7 +118,7 @@ public DDAgentFeaturesDiscovery( this.agentBaseUrl = agentUrl; this.metricsEnabled = metricsEnabled; this.ignoreAgentVersionForStats = ignoreAgentVersionForStats; - this.protocolVersion = protocolVersion != null ? protocolVersion : V0_4; + this.protocolVersion = protocolVersion != null ? protocolVersion : V1_0; this.discoveryTimer = monitoring.newTimer("trace.agent.discovery.time"); this.discoveryState = new State(); } diff --git a/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy b/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy index 8bcda6eb811..6fae9ca5f40 100644 --- a/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy +++ b/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy @@ -87,7 +87,7 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { V1_0 | V1_ENDPOINT } - def "null protocol version falls back to v0.4 trace endpoints"() { + def "null protocol version falls back to v1.0 trace endpoints"() { setup: OkHttpClient client = Mock(OkHttpClient) DDAgentFeaturesDiscovery features = @@ -98,9 +98,10 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { then: 1 * client.newCall({ Request request -> request.url().toString() == "http://localhost:8125/info" }) >> { Request request -> infoResponse(request, "{}") } + 1 * client.newCall({ Request request -> request.url().toString() == "http://localhost:8125/v1.0/traces" }) >> { Request request -> success(request) } 0 * client.newCall({ Request request -> request.url().toString() == "http://localhost:8125/v0.5/traces" }) >> { Request request -> success(request) } - 1 * client.newCall({ Request request -> request.url().toString() == "http://localhost:8125/v0.4/traces" }) >> { Request request -> success(request) } - features.getTraceEndpoint() == V04_ENDPOINT + 0 * client.newCall({ Request request -> request.url().toString() == "http://localhost:8125/v0.4/traces" }) >> { Request request -> success(request) } + features.getTraceEndpoint() == V1_ENDPOINT 0 * _ } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 991c6fd6937..0e88b3cbf43 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -92,7 +92,7 @@ public final class ConfigDefaults { Arrays.asList("user.id", "session.id", "account.id"); static final boolean DEFAULT_JMX_FETCH_ENABLED = true; - static final String DEFAULT_TRACE_AGENT_PROTOCOL_VERSION = ProtocolVersion.V0_4.asConfigValue(); + static final String DEFAULT_TRACE_AGENT_PROTOCOL_VERSION = ProtocolVersion.V1_0.asConfigValue(); static final boolean DEFAULT_CLIENT_IP_ENABLED = false; From 0f351ca09554db2209a950aa3ac70bbd9a975272 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 30 Jul 2026 10:57:31 -0400 Subject: [PATCH 03/11] Debug SSI system-tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ca5a947885d..0039f2bee31 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1475,7 +1475,7 @@ publishing-gate: configure_system_tests: variables: - SYSTEM_TESTS_REF: "main" # system tests are pinned on release branches only + SYSTEM_TESTS_REF: "alexeyk/fix-v1-appsec-event-metrics-validation" # system tests are pinned on release branches only SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" create_key: From 707372a76063d40d253c2898939bc7c572e22ca6 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 30 Jul 2026 11:41:03 -0400 Subject: [PATCH 04/11] Debug SSI system-tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0039f2bee31..59e82393324 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1475,7 +1475,7 @@ publishing-gate: configure_system_tests: variables: - SYSTEM_TESTS_REF: "alexeyk/fix-v1-appsec-event-metrics-validation" # system tests are pinned on release branches only + SYSTEM_TESTS_REF: "alexeyk/fix-v1-appsec-event-metrics-validation" # system-tests are pinned on release branches only SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" create_key: From f707cb8982124ea87fc58afa519b364ce457fcc4 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 30 Jul 2026 12:39:24 -0400 Subject: [PATCH 05/11] Debug SSI system-tests: actual fix --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 59e82393324..0039f2bee31 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1475,7 +1475,7 @@ publishing-gate: configure_system_tests: variables: - SYSTEM_TESTS_REF: "alexeyk/fix-v1-appsec-event-metrics-validation" # system-tests are pinned on release branches only + SYSTEM_TESTS_REF: "alexeyk/fix-v1-appsec-event-metrics-validation" # system tests are pinned on release branches only SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" create_key: From 7760deecc423d20dd7cb0231e71cecc5cf83d4ac Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 31 Jul 2026 16:15:57 -0400 Subject: [PATCH 06/11] Test with systemt-test from master. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f7f8b5f7e0c..fed9f25df92 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1475,7 +1475,7 @@ publishing-gate: configure_system_tests: variables: - SYSTEM_TESTS_REF: "alexeyk/fix-v1-appsec-event-metrics-validation" # system tests are pinned on release branches only + SYSTEM_TESTS_REF: "main" # system tests are pinned on release branches only SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" create_key: From 314a2c6ae1aa320f165c15bde9b0559c1ee3aa38 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 31 Jul 2026 20:40:26 -0400 Subject: [PATCH 07/11] Test the latest fix in system-tests. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fed9f25df92..dfe0bcbfd46 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1475,7 +1475,7 @@ publishing-gate: configure_system_tests: variables: - SYSTEM_TESTS_REF: "main" # system tests are pinned on release branches only + SYSTEM_TESTS_REF: "alexeyk/fix-v1-trace-id-comparison" # system tests are pinned on release branches only SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" create_key: From ed1e4321b37c7b88cc18a7019456b2dae266bd5b Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 31 Jul 2026 21:18:35 -0400 Subject: [PATCH 08/11] Test the latest fix in system-tests. --- .github/workflows/run-system-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-system-tests.yaml b/.github/workflows/run-system-tests.yaml index 4b0ad663659..b38faee1cca 100644 --- a/.github/workflows/run-system-tests.yaml +++ b/.github/workflows/run-system-tests.yaml @@ -62,7 +62,7 @@ jobs: needs: - build # If you change the following comment, update the pattern in the update_system_test_reference.sh script to match. - uses: DataDog/system-tests/.github/workflows/system-tests.yml@main # system tests are pinned on release branches only + uses: DataDog/system-tests/.github/workflows/system-tests.yml@alexeyk/fix-v1-trace-id-comparison # system tests are pinned on release branches only permissions: contents: read id-token: write @@ -70,7 +70,7 @@ jobs: with: library: java # If you change the following comment, update the pattern in the update_system_test_reference.sh script to match. - ref: "main" # system tests are pinned on release branches only + ref: "alexeyk/fix-v1-trace-id-comparison" # system tests are pinned on release branches only binaries_artifact: binaries desired_execution_time: 900 # 15 minutes scenarios_groups: tracer-release From a69117d3fce1a33f1c7341c19a3d051bc457325e Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 3 Aug 2026 09:06:48 -0400 Subject: [PATCH 09/11] Test v1 on master. --- .github/workflows/run-system-tests.yaml | 4 ++-- .gitlab-ci.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-system-tests.yaml b/.github/workflows/run-system-tests.yaml index b38faee1cca..4b0ad663659 100644 --- a/.github/workflows/run-system-tests.yaml +++ b/.github/workflows/run-system-tests.yaml @@ -62,7 +62,7 @@ jobs: needs: - build # If you change the following comment, update the pattern in the update_system_test_reference.sh script to match. - uses: DataDog/system-tests/.github/workflows/system-tests.yml@alexeyk/fix-v1-trace-id-comparison # system tests are pinned on release branches only + uses: DataDog/system-tests/.github/workflows/system-tests.yml@main # system tests are pinned on release branches only permissions: contents: read id-token: write @@ -70,7 +70,7 @@ jobs: with: library: java # If you change the following comment, update the pattern in the update_system_test_reference.sh script to match. - ref: "alexeyk/fix-v1-trace-id-comparison" # system tests are pinned on release branches only + ref: "main" # system tests are pinned on release branches only binaries_artifact: binaries desired_execution_time: 900 # 15 minutes scenarios_groups: tracer-release diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dfe0bcbfd46..fed9f25df92 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1475,7 +1475,7 @@ publishing-gate: configure_system_tests: variables: - SYSTEM_TESTS_REF: "alexeyk/fix-v1-trace-id-comparison" # system tests are pinned on release branches only + SYSTEM_TESTS_REF: "main" # system tests are pinned on release branches only SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" create_key: From aabb490c8aa8980e24a0a69d4845124fbf1dc05a Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 14 Oct 2026 09:12:39 -0400 Subject: [PATCH 10/11] Preserve v1 span events in test-agent JSON decoding. --- .../agent/decoder/json/raw/MessageJson.java | 87 +++++++++++++++++++ .../test/agent/decoder/json/raw/SpanJson.java | 22 +++++ .../test/agent/decoder/JsonDecoderTest.java | 53 +++++++++++ 3 files changed, 162 insertions(+) diff --git a/utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/json/raw/MessageJson.java b/utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/json/raw/MessageJson.java index 42e8971cf06..dae87d04c92 100644 --- a/utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/json/raw/MessageJson.java +++ b/utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/json/raw/MessageJson.java @@ -1,10 +1,12 @@ package datadog.trace.test.agent.decoder.json.raw; +import static java.lang.annotation.RetentionPolicy.RUNTIME; import static java.util.Collections.unmodifiableList; import com.squareup.moshi.FromJson; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.JsonDataException; +import com.squareup.moshi.JsonQualifier; import com.squareup.moshi.JsonReader; import com.squareup.moshi.JsonWriter; import com.squareup.moshi.Moshi; @@ -14,12 +16,15 @@ import datadog.trace.test.agent.decoder.DecodedSpan; import datadog.trace.test.agent.decoder.DecodedTrace; import java.io.IOException; +import java.lang.annotation.Retention; import java.lang.reflect.Type; import java.math.BigInteger; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import okio.Buffer; +import okio.BufferedSink; /** * MessageJson decodes a JSON trace payload — a JSON array of traces, each a JSON array of spans in @@ -35,6 +40,7 @@ public final class MessageJson implements DecodedMessage { new Moshi.Builder() .add(new MetricNumberAdapter()) .add(new MetaStructObjectAdapter()) + .add(new RawJsonAdapter()) .build() .adapter(LIST_OF_TRACES); @@ -74,6 +80,7 @@ public static MessageJson fromJson(String json) { + span); } span.resolveLinks(); + span.resolveEvents(); decodedSpans.add(span); } traces.add(new TraceJson(decodedSpans)); @@ -188,6 +195,86 @@ private static Number parseNumber(String literal) { } } + /** Marks a {@code String} field that holds a JSON value as compact JSON text. */ + @Retention(RUNTIME) + @JsonQualifier + @interface RawJson {} + + /** + * Decodes a {@link RawJson} field by copying the JSON value token by token into compact JSON + * text. Number literals are copied verbatim rather than materialized as {@code Double}, so values + * above 2^53 or beyond {@code long} range, and fractional values, keep their exact textual form. + * A JSON {@code null} decodes to a {@code null} field. + */ + static final class RawJsonAdapter { + @FromJson + @RawJson + String fromJson(JsonReader reader) throws IOException { + if (reader.peek() == JsonReader.Token.NULL) { + return reader.nextNull(); + } + Buffer buffer = new Buffer(); + JsonWriter writer = JsonWriter.of(buffer); + // Keep null members ("key":null) rather than dropping them. + writer.setSerializeNulls(true); + copyValue(reader, writer); + writer.flush(); + return buffer.readUtf8(); + } + + @ToJson + void toJson(JsonWriter writer, @RawJson String value) throws IOException { + if (value == null) { + writer.nullValue(); + return; + } + try (BufferedSink sink = writer.valueSink()) { + sink.writeUtf8(value); + } + } + + private static void copyValue(JsonReader reader, JsonWriter writer) throws IOException { + switch (reader.peek()) { + case BEGIN_OBJECT: + reader.beginObject(); + writer.beginObject(); + while (reader.hasNext()) { + writer.name(reader.nextName()); + copyValue(reader, writer); + } + reader.endObject(); + writer.endObject(); + break; + case BEGIN_ARRAY: + reader.beginArray(); + writer.beginArray(); + while (reader.hasNext()) { + copyValue(reader, writer); + } + reader.endArray(); + writer.endArray(); + break; + case STRING: + writer.value(reader.nextString()); + break; + case NUMBER: + try (BufferedSink sink = writer.valueSink()) { + sink.writeUtf8(reader.nextString()); + } + break; + case BOOLEAN: + writer.value(reader.nextBoolean()); + break; + case NULL: + reader.nextNull(); + writer.nullValue(); + break; + default: + throw new JsonDataException("Unexpected JSON token: " + reader.peek()); + } + } + } + @Override public List getTraces() { return this.traces; diff --git a/utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/json/raw/SpanJson.java b/utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/json/raw/SpanJson.java index 8c59fd6835b..7cee1cf20cf 100644 --- a/utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/json/raw/SpanJson.java +++ b/utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/json/raw/SpanJson.java @@ -9,6 +9,7 @@ import datadog.trace.test.agent.decoder.DecodedSpanLink; import datadog.trace.test.agent.decoder.DecodedSpanLinks; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -53,6 +54,12 @@ public final class SpanJson implements DecodedSpan { @Json(name = "span_links") List spanLinks; + // Only present in V1 payload. Kept as compact JSON text (number literals verbatim, see + // MessageJson.RawJsonAdapter) and exposed as the events meta tag by #resolveEvents + @Json(name = "span_events") + @MessageJson.RawJson + String spanEvents; + // Decoded from span links meta tag or #spanLinks // transient so Moshi does not try to decode it transient List links; @@ -146,6 +153,21 @@ void resolveLinks() { this.links = unmodifiableList(decoded); } + /** + * Exposes the {@code span_events} field, when the agent filled it (the tracer submitted a {@code + * v1.0} payload), as the {@code events} meta tag, replacing any such tag. The value is the + * agent's JSON re-emitted compactly; its member order and attribute shape are the agent's, not + * those of the tag the tracer writes for older protocols. + */ + void resolveEvents() { + if (this.spanEvents != null) { + if (this.meta == null) { + this.meta = new HashMap<>(); + } + this.meta.put("events", this.spanEvents); + } + } + @Override public String toString() { return "SpanJson{" diff --git a/utils/test-agent-utils/decoder/src/test/java/datadog/trace/test/agent/decoder/JsonDecoderTest.java b/utils/test-agent-utils/decoder/src/test/java/datadog/trace/test/agent/decoder/JsonDecoderTest.java index 78d60bf8e0f..33e5e349881 100644 --- a/utils/test-agent-utils/decoder/src/test/java/datadog/trace/test/agent/decoder/JsonDecoderTest.java +++ b/utils/test-agent-utils/decoder/src/test/java/datadog/trace/test/agent/decoder/JsonDecoderTest.java @@ -26,6 +26,59 @@ class JsonDecoderTest { // highlighting). private static final String TWO_TRACES = loadJson("/two-traces.json"); + @Test + void exposesNativeSpanEventsWithNumberLiteralsVerbatim() { + String events = + "[{\"name\":\"configuration-loaded\",\"time_unix_nano\":1791902357478000001," + + "\"attributes\":{\"source\":\"file\",\"count\":3,\"cached\":false," + + "\"ratio\":0.123456789012,\"huge\":18446744073709551616,\"missing\":null," + + "\"list\":[1,2.5,\"three\"]}}]"; + DecodedSpan span = decodeSpanWithEvents("\"span_events\":" + events); + + assertEquals(events, span.getMeta().get("events")); + } + + @Test + void compactsNativeSpanEvents() { + DecodedSpan span = + decodeSpanWithEvents( + "\"span_events\": [ {\"name\": \"a \\\"quoted\\\" name\", \"time_unix_nano\": 1e3} ]"); + + assertEquals( + "[{\"name\":\"a \\\"quoted\\\" name\",\"time_unix_nano\":1e3}]", + span.getMeta().get("events")); + } + + @Test + void preservesLegacySpanEventsTag() { + String events = "[{\"name\":\"configuration-loaded\"}]"; + DecodedSpan span = + decodeSpanWithEvents( + "\"meta\":{\"events\":\"[{\\\"name\\\":\\\"configuration-loaded\\\"}]\"}"); + + assertEquals(events, span.getMeta().get("events")); + } + + @Test + void nativeSpanEventsTakePrecedenceAndEmptyEventsStayEmpty() { + DecodedSpan span = decodeSpanWithEvents("\"meta\":{\"events\":\"legacy\"},\"span_events\":[]"); + + assertEquals("[]", span.getMeta().get("events")); + assertNull(decodeSpanWithEvents("\"span_events\":null").getMeta().get("events")); + assertNull(decodeSpanWithEvents("\"meta\":{}").getMeta().get("events")); + } + + private static DecodedSpan decodeSpanWithEvents(String fields) { + return Decoder.decodeJson( + "[[{\"name\":\"test\",\"trace_id\":1,\"span_id\":2,\"start\":3,\"duration\":4," + + fields + + "}]]") + .getTraces() + .get(0) + .getSpans() + .get(0); + } + @Test void decodesTraceAndSpanStructure() { List traces = Decoder.decodeJson(TWO_TRACES).getTraces(); From c385b63bd89dfea59cde68105f1ec643e82f80cd Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 14 Oct 2026 14:00:57 -0400 Subject: [PATCH 11/11] Decode v1 span events in the msgpack test-agent decoder. SpanV1 skipped the span events field, so spans decoded from v1 payloads lost their events, and the blind skip consumed streaming strings without registering them, desynchronizing the string table. Decode the events through the table-aware helpers into the same events meta tag the tracer writes for v0.4, so both protocols decode to the same tag. Also split the JSON decoder span events precedence/empty test into focused tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../test/agent/decoder/v1/raw/SpanV1.java | 120 +++++++++++++++++- .../test/agent/decoder/JsonDecoderTest.java | 17 ++- .../test/agent/decoder/v1/raw/SpanV1Test.java | 82 +++++++++++- 3 files changed, 209 insertions(+), 10 deletions(-) diff --git a/utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/v1/raw/SpanV1.java b/utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/v1/raw/SpanV1.java index c4ec1e4b6b9..7baeaf2572b 100644 --- a/utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/v1/raw/SpanV1.java +++ b/utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/v1/raw/SpanV1.java @@ -5,14 +5,17 @@ import static java.util.Collections.unmodifiableList; import static java.util.Collections.unmodifiableMap; +import com.squareup.moshi.JsonWriter; import datadog.trace.test.agent.decoder.DecodedSpan; import datadog.trace.test.agent.decoder.DecodedSpanLink; import datadog.trace.test.agent.decoder.DecodedSpanLinks; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import okio.Buffer; import org.msgpack.core.MessageFormat; import org.msgpack.core.MessagePack; import org.msgpack.core.MessageUnpacker; @@ -59,6 +62,14 @@ public class SpanV1 implements DecodedSpan { static final int LINK_FIELD_TRACE_STATE = 4; static final int LINK_FIELD_TRACE_FLAGS = 5; + // Span event field IDs (from TraceMapperV1.encodeSpanEvents) + static final int EVENT_FIELD_TIME = 1; + static final int EVENT_FIELD_NAME = 2; + static final int EVENT_FIELD_ATTRIBUTES = 3; + + // The meta tag the tracer encodes span events into for protocols without native events + static final String SPAN_EVENTS_TAG = "events"; + // Attribute value types static final int STRING_VALUE_TYPE = 1; static final int BOOL_VALUE_TYPE = 2; @@ -172,10 +183,10 @@ static SpanV1 unpack(MessageUnpacker unpacker, List stringTable, long tr links = unpackSpanLinks(unpacker, stringTable); break; case SPAN_FIELD_SPAN_EVENTS: - // TODO Span events are not decoded yet. This blind skip also consumes any streaming - // string the events introduced without registering it, desynchronizing the string - // table for the rest of the payload (see unpackSpanLinks for the table-aware pattern). - unpacker.skipValue(); + String events = unpackSpanEvents(unpacker, stringTable); + if (events != null) { + meta.put(SPAN_EVENTS_TAG, events); + } break; case SPAN_FIELD_ENV: env = unpackStreamingString(unpacker, stringTable); @@ -396,6 +407,107 @@ static List unpackSpanLinks(MessageUnpacker unpacker, ListEvent names and attributes are streaming strings, so they are read through the table-aware + * helpers to keep every later string index in the payload in sync. + * + * @param unpacker The message unpacker. + * @param stringTable The shared string table for streaming string decoding. + * @return The JSON tag value, {@code null} if there are no events. + * @throws IOException If unpacking fails. + */ + static String unpackSpanEvents(MessageUnpacker unpacker, List stringTable) + throws IOException { + int eventCount = unpacker.unpackArrayHeader(); + if (eventCount == 0) { + return null; + } + List> events = new ArrayList<>(eventCount); + for (int i = 0; i < eventCount; i++) { + int fieldCount = unpacker.unpackMapHeader(); + long time = 0; + String name = ""; + Map attributes = new LinkedHashMap<>(); + for (int field = 0; field < fieldCount; field++) { + switch (unpacker.unpackInt()) { + case EVENT_FIELD_TIME: + time = unpacker.unpackLong(); + break; + case EVENT_FIELD_NAME: + name = unpackStreamingString(unpacker, stringTable); + break; + case EVENT_FIELD_ATTRIBUTES: + forEachAttribute( + unpacker, + stringTable, + (attributeUnpacker, table, key, valueType) -> + attributes.put( + key, unpackEventAttributeValue(attributeUnpacker, table, valueType))); + break; + default: + // Unknown event field; numeric or binary, so it cannot hold a streaming string. + unpacker.skipValue(); + break; + } + } + Map event = new LinkedHashMap<>(); + event.put("time_unix_nano", time); + event.put("name", name); + if (!attributes.isEmpty()) { + event.put("attributes", attributes); + } + events.add(event); + } + Buffer buffer = new Buffer(); + JsonWriter writer = JsonWriter.of(buffer); + // Unsupported attribute values are written as null and non-finite doubles as bare literals, + // as the tracer's events tag does. + writer.setSerializeNulls(true); + writer.setLenient(true); + writer.jsonValue(events); + writer.flush(); + return buffer.readUtf8(); + } + + /** + * Unpacks an event attribute value: a string, boolean, double, long, or a list of them. Value + * types the tracer never writes for events decode to {@code null}. + */ + private static Object unpackEventAttributeValue( + MessageUnpacker unpacker, List stringTable, int valueType) throws IOException { + switch (valueType) { + case STRING_VALUE_TYPE: + return unpackStreamingString(unpacker, stringTable); + case BOOL_VALUE_TYPE: + return unpacker.unpackBoolean(); + case FLOAT_VALUE_TYPE: + return unpacker.unpackDouble(); + case INT_VALUE_TYPE: + return unpacker.unpackLong(); + case ARRAY_VALUE_TYPE: + int arraySize = unpacker.unpackArrayHeader(); + // Array contains pairs (type, value), so size must be even + if (arraySize % 2 != 0) { + throw new IllegalArgumentException( + "Event attribute array size must be even, got: " + arraySize); + } + List items = new ArrayList<>(arraySize / 2); + for (int i = 0; i < arraySize / 2; i++) { + items.add(unpackEventAttributeValue(unpacker, stringTable, unpacker.unpackInt())); + } + return items; + default: + unpacker.skipValue(); + return null; + } + } + /** * Unpacks an identifier the writer emitted as an unsigned 64-bit value. * diff --git a/utils/test-agent-utils/decoder/src/test/java/datadog/trace/test/agent/decoder/JsonDecoderTest.java b/utils/test-agent-utils/decoder/src/test/java/datadog/trace/test/agent/decoder/JsonDecoderTest.java index 33e5e349881..20a7d55b59a 100644 --- a/utils/test-agent-utils/decoder/src/test/java/datadog/trace/test/agent/decoder/JsonDecoderTest.java +++ b/utils/test-agent-utils/decoder/src/test/java/datadog/trace/test/agent/decoder/JsonDecoderTest.java @@ -60,10 +60,21 @@ void preservesLegacySpanEventsTag() { } @Test - void nativeSpanEventsTakePrecedenceAndEmptyEventsStayEmpty() { - DecodedSpan span = decodeSpanWithEvents("\"meta\":{\"events\":\"legacy\"},\"span_events\":[]"); + void nativeSpanEventsReplaceLegacySpanEventsTag() { + DecodedSpan span = + decodeSpanWithEvents( + "\"meta\":{\"events\":\"legacy\"},\"span_events\":[{\"name\":\"native\"}]"); + + assertEquals("[{\"name\":\"native\"}]", span.getMeta().get("events")); + } - assertEquals("[]", span.getMeta().get("events")); + @Test + void emptyNativeSpanEventsDecodeToAnEmptyArray() { + assertEquals("[]", decodeSpanWithEvents("\"span_events\":[]").getMeta().get("events")); + } + + @Test + void nullOrAbsentNativeSpanEventsLeaveNoEventsTag() { assertNull(decodeSpanWithEvents("\"span_events\":null").getMeta().get("events")); assertNull(decodeSpanWithEvents("\"meta\":{}").getMeta().get("events")); } diff --git a/utils/test-agent-utils/decoder/src/test/java/datadog/trace/test/agent/decoder/v1/raw/SpanV1Test.java b/utils/test-agent-utils/decoder/src/test/java/datadog/trace/test/agent/decoder/v1/raw/SpanV1Test.java index 03e4ece9d6e..1c9f6d50ced 100644 --- a/utils/test-agent-utils/decoder/src/test/java/datadog/trace/test/agent/decoder/v1/raw/SpanV1Test.java +++ b/utils/test-agent-utils/decoder/src/test/java/datadog/trace/test/agent/decoder/v1/raw/SpanV1Test.java @@ -1,7 +1,9 @@ package datadog.trace.test.agent.decoder.v1.raw; +import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import datadog.trace.test.agent.decoder.DecodedSpanLink; import java.io.IOException; @@ -14,9 +16,9 @@ import org.msgpack.core.MessageUnpacker; /** - * Tests the {@code v1.0} span decoder against identifiers whose high bit is set. {@code - * TraceMapperV1} writes span identifiers with {@code writeUnsignedLong}, so those arrive as msgpack - * {@code UINT64} — a shape the {@code sample_v1.msgpack} fixture never produces. + * Tests the {@code v1.0} span decoder against shapes the {@code sample_v1.msgpack} fixture never + * produces: identifiers whose high bit is set ({@code TraceMapperV1} writes span identifiers with + * {@code writeUnsignedLong}, so those arrive as msgpack {@code UINT64}) and span events. */ class SpanV1Test { // 0xfedcba9876543210: an unsigned 64-bit identifier above Long.MAX_VALUE. @@ -71,6 +73,80 @@ void noSpanLinksDecodeToAnEmptyList() throws IOException { assertEquals(emptyList(), SpanV1.unpackSpanLinks(unpacker(packer), new ArrayList<>())); } + @Test + void decodesSpanEventsIntoTheEventsTag() throws IOException { + MessageBufferPacker packer = MessagePack.newDefaultBufferPacker(); + packer.packArrayHeader(2); + // First event: every attribute value type, introducing new streaming strings. + packer.packMapHeader(3); + packer.packInt(SpanV1.EVENT_FIELD_TIME); + packer.packLong(1791902357478000001L); + packer.packInt(SpanV1.EVENT_FIELD_NAME); + packer.packString("configuration-loaded"); // string table index 0 + packer.packInt(SpanV1.EVENT_FIELD_ATTRIBUTES); + packer.packArrayHeader(5 * 3); + packer.packString("source").packInt(SpanV1.STRING_VALUE_TYPE).packString("file \"local\""); + packer.packString("count").packInt(SpanV1.INT_VALUE_TYPE).packLong(3); + packer.packString("ratio").packInt(SpanV1.FLOAT_VALUE_TYPE).packDouble(2.5); + packer.packString("cached").packInt(SpanV1.BOOL_VALUE_TYPE).packBoolean(false); + packer.packString("tags").packInt(SpanV1.ARRAY_VALUE_TYPE).packArrayHeader(2 * 2); + packer.packInt(SpanV1.STRING_VALUE_TYPE).packInt(0); // reference to "configuration-loaded" + packer.packInt(SpanV1.INT_VALUE_TYPE).packLong(1); + // Second event: name by reference, no attributes. + packer.packMapHeader(3); + packer.packInt(SpanV1.EVENT_FIELD_TIME); + packer.packLong(2); + packer.packInt(SpanV1.EVENT_FIELD_NAME); + packer.packInt(0); + packer.packInt(SpanV1.EVENT_FIELD_ATTRIBUTES); + packer.packArrayHeader(0); + List stringTable = new ArrayList<>(); + + String events = SpanV1.unpackSpanEvents(unpacker(packer), stringTable); + + assertEquals( + "[{\"time_unix_nano\":1791902357478000001,\"name\":\"configuration-loaded\"," + + "\"attributes\":{\"source\":\"file \\\"local\\\"\",\"count\":3,\"ratio\":2.5," + + "\"cached\":false,\"tags\":[\"configuration-loaded\",1]}}," + + "{\"time_unix_nano\":2,\"name\":\"configuration-loaded\"}]", + events); + assertEquals( + asList( + "configuration-loaded", "source", "file \"local\"", "count", "ratio", "cached", "tags"), + stringTable, + "event strings registered in the string table"); + } + + @Test + void spanEventsKeepTheStringTableInSync() throws IOException { + MessageBufferPacker packer = MessagePack.newDefaultBufferPacker(); + packer.packMapHeader(2); + packer.packInt(SpanV1.SPAN_FIELD_SPAN_EVENTS); + packer.packArrayHeader(1); + packer.packMapHeader(1); + packer.packInt(SpanV1.EVENT_FIELD_NAME); + packer.packString("event-name"); // string table index 0 + packer.packInt(SpanV1.SPAN_FIELD_SERVICE); + packer.packInt(0); // reference to the string the event introduced + + SpanV1 span = SpanV1.unpack(unpacker(packer), new ArrayList<>()); + + assertEquals("[{\"time_unix_nano\":0,\"name\":\"event-name\"}]", span.getMeta().get("events")); + assertEquals("event-name", span.getService()); + } + + @Test + void noSpanEventsLeaveNoEventsTag() throws IOException { + MessageBufferPacker packer = MessagePack.newDefaultBufferPacker(); + packer.packMapHeader(1); + packer.packInt(SpanV1.SPAN_FIELD_SPAN_EVENTS); + packer.packArrayHeader(0); + + SpanV1 span = SpanV1.unpack(unpacker(packer), new ArrayList<>()); + + assertFalse(span.getMeta().containsKey("events")); + } + private static long spanIdOf(BigInteger id) throws IOException { MessageBufferPacker packer = MessagePack.newDefaultBufferPacker(); packer.packMapHeader(1);