Preserve v1 span events in test-agent decoders

dd-smoke-tests/opentelemetry/src/test/java/datadog/smoketest/OpenTelemetrySmokeTest.java

@@ -6,17 +6,37 @@
 import static datadog.smoketest.trace.SpanMatcher.span;
 import static datadog.smoketest.trace.TraceMatcher.trace;
 import static datadog.trace.test.junit.utils.assertions.Matchers.validates;
+import static java.util.Arrays.asList;
+import static java.util.Arrays.stream;
 import static java.util.Comparator.comparing;
 import static java.util.concurrent.TimeUnit.SECONDS;
 
 import datadog.smoketest.trace.SpanMatcher;
 import datadog.smoketest.trace.TraceMatcher.Options;
+import datadog.trace.api.ProtocolVersion;
 import datadog.trace.test.agent.decoder.DecodedSpan;
 import java.io.File;
+import java.util.List;
 import java.util.function.UnaryOperator;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.ClassTemplate;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.extension.AfterClassTemplateInvocationCallback;
+import org.junit.jupiter.api.extension.BeforeClassTemplateInvocationCallback;
+import org.junit.jupiter.api.extension.ClassTemplateInvocationContext;
+import org.junit.jupiter.api.extension.ClassTemplateInvocationContextProvider;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TestInstancePostProcessor;
 
+/**
+ * Runs the OpenTelemetry app once per trace agent protocol: span events and links travel as meta
+ * tags with {@code v0.4} and {@code v0.5} but as native span fields with {@code v1.0}, and must
+ * decode alike.
+ */
+@ClassTemplate
+@ExtendWith(OpenTelemetrySmokeTest.AppPerProtocol.class)
 class OpenTelemetrySmokeTest {
   private static final UnaryOperator<Options> SORT_BY_RESOURCE_NAME =
       options -> options.sort(comparing(DecodedSpan::getResource));
@@ -26,14 +46,8 @@ class OpenTelemetrySmokeTest {
   private static final String APPLICATION_JAR =
       System.getProperty("datadog.smoketest.shadowJar.path");
 
-  @RegisterExtension
-  static final SmokeCliApp app =
-      SmokeCliApp.named("opentelemetry")
-          .jar(APPLICATION_JAR)
-          .jvmArgs("-Ddd.trace.otel.enabled=true")
-          .workingDirectory(WORKING_DIRECTORY)
-          .backend(testAgent())
-          .build();
+  /** The app of the current protocol, set by {@link AppLifecycle}. */
+  private SmokeCliApp app;
 
   @Test
   void receivesTraces() {
@@ -82,11 +96,86 @@ void receivesTraces() {
     app.assertCompletesWithValue(TIMEOUT_SECONDS, SECONDS, 0);
   }
 
+  private static SmokeCliApp app(ProtocolVersion protocol) {
+    return SmokeCliApp.named("opentelemetry")
+        .jar(APPLICATION_JAR)
+        .jvmArgs(
+            "-Ddd.trace.otel.enabled=true",
+            "-Ddd.trace.agent.protocol.version=" + protocol.asConfigValue())
+        .workingDirectory(WORKING_DIRECTORY)
+        .backend(testAgent())
+        .build();
+  }
+
   private static SpanMatcher shard(int shard) {
     return span()
         .operationName("internal")
         .resourceName("shard-" + shard)
         .childOfIndex(2)
         .metric("batch.shard", validates(value -> value.intValue() == shard));
   }
+
+  /**
+   * Runs the class once per {@link ProtocolVersion}, each invocation with its own app. Class-level
+   * callbacks such as {@code BeforeAllCallback} run once per class template, not per invocation, so
+   * {@link AppLifecycle} launches and stops the app around its invocation instead.
+   */
+  static final class AppPerProtocol implements ClassTemplateInvocationContextProvider {
+    @Override
+    public boolean supportsClassTemplate(ExtensionContext context) {
+      return true;
+    }
+
+    @Override
+    public Stream<ClassTemplateInvocationContext> provideClassTemplateInvocationContexts(
+        ExtensionContext context) {
+      return stream(ProtocolVersion.values()).map(AppPerProtocol::invocation);
+    }
+
+    private static ClassTemplateInvocationContext invocation(ProtocolVersion protocol) {
+      SmokeCliApp app = app(protocol);
+      return new ClassTemplateInvocationContext() {
+        @Override
+        public String getDisplayName(int invocationIndex) {
+          return "protocol " + protocol.asConfigValue();
+        }
+
+        @Override
+        public List<Extension> getAdditionalExtensions() {
+          // The app itself for its per-test-method callbacks.
+          return asList(app, new AppLifecycle(app));
+        }
+      };
+    }
+  }
+
+  /**
+   * Launches and stops an app around a class template invocation, and sets it on each test instance
+   * of that invocation.
+   */
+  static final class AppLifecycle
+      implements BeforeClassTemplateInvocationCallback,
+          AfterClassTemplateInvocationCallback,
+          TestInstancePostProcessor {
+    private final SmokeCliApp app;
+
+    AppLifecycle(SmokeCliApp app) {
+      this.app = app;
+    }
+
+    @Override
+    public void beforeClassTemplateInvocation(ExtensionContext context) throws Exception {
+      this.app.beforeAll(context);
+    }
+
+    @Override
+    public void afterClassTemplateInvocation(ExtensionContext context) {
+      this.app.afterAll(context);
+    }
+
+    @Override
+    public void postProcessTestInstance(Object testInstance, ExtensionContext context) {
+      ((OpenTelemetrySmokeTest) testInstance).app = this.app;
+    }
+  }
 }

utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/json/raw/MessageJson.java

@@ -1,10 +1,12 @@
 package datadog.trace.test.agent.decoder.json.raw;
 
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
 import static java.util.Collections.unmodifiableList;
 
 import com.squareup.moshi.FromJson;
 import com.squareup.moshi.JsonAdapter;
 import com.squareup.moshi.JsonDataException;
+import com.squareup.moshi.JsonQualifier;
 import com.squareup.moshi.JsonReader;
 import com.squareup.moshi.JsonWriter;
 import com.squareup.moshi.Moshi;
@@ -14,12 +16,15 @@
 import datadog.trace.test.agent.decoder.DecodedSpan;
 import datadog.trace.test.agent.decoder.DecodedTrace;
 import java.io.IOException;
+import java.lang.annotation.Retention;
 import java.lang.reflect.Type;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import okio.Buffer;
+import okio.BufferedSink;
 
 /**
  * MessageJson decodes a JSON trace payload — a JSON array of traces, each a JSON array of spans in
@@ -35,6 +40,7 @@ public final class MessageJson implements DecodedMessage {
       new Moshi.Builder()
           .add(new MetricNumberAdapter())
           .add(new MetaStructObjectAdapter())
+          .add(new RawJsonAdapter())
           .build()
           .adapter(LIST_OF_TRACES);
 
@@ -74,6 +80,7 @@ public static MessageJson fromJson(String json) {
                     + span);
           }
           span.resolveLinks();
+          span.resolveEvents();
           decodedSpans.add(span);
         }
         traces.add(new TraceJson(decodedSpans));
@@ -188,6 +195,86 @@ private static Number parseNumber(String literal) {
     }
   }
 
+  /** Marks a {@code String} field that holds a JSON value as compact JSON text. */
+  @Retention(RUNTIME)
+  @JsonQualifier
+  @interface RawJson {}
+
+  /**
+   * Decodes a {@link RawJson} field by copying the JSON value token by token into compact JSON
+   * text. Number literals are copied verbatim rather than materialized as {@code Double}, so values
+   * above 2^53 or beyond {@code long} range, and fractional values, keep their exact textual form.
+   * A JSON {@code null} decodes to a {@code null} field.
+   */
+  static final class RawJsonAdapter {
+    @FromJson
+    @RawJson
+    String fromJson(JsonReader reader) throws IOException {
+      if (reader.peek() == JsonReader.Token.NULL) {
+        return reader.nextNull();
+      }
+      Buffer buffer = new Buffer();
+      JsonWriter writer = JsonWriter.of(buffer);
+      // Keep null members ("key":null) rather than dropping them.
+      writer.setSerializeNulls(true);
+      copyValue(reader, writer);
+      writer.flush();
+      return buffer.readUtf8();
+    }
+
+    @ToJson
+    void toJson(JsonWriter writer, @RawJson String value) throws IOException {
+      if (value == null) {
+        writer.nullValue();
+        return;
+      }
+      try (BufferedSink sink = writer.valueSink()) {
+        sink.writeUtf8(value);
+      }
+    }
+
+    private static void copyValue(JsonReader reader, JsonWriter writer) throws IOException {
+      switch (reader.peek()) {
+        case BEGIN_OBJECT:
+          reader.beginObject();
+          writer.beginObject();
+          while (reader.hasNext()) {
+            writer.name(reader.nextName());
+            copyValue(reader, writer);
+          }
+          reader.endObject();
+          writer.endObject();
+          break;
+        case BEGIN_ARRAY:
+          reader.beginArray();
+          writer.beginArray();
+          while (reader.hasNext()) {
+            copyValue(reader, writer);
+          }
+          reader.endArray();
+          writer.endArray();
+          break;
+        case STRING:
+          writer.value(reader.nextString());
+          break;
+        case NUMBER:
+          try (BufferedSink sink = writer.valueSink()) {
+            sink.writeUtf8(reader.nextString());
+          }
+          break;
+        case BOOLEAN:
+          writer.value(reader.nextBoolean());
+          break;
+        case NULL:
+          reader.nextNull();
+          writer.nullValue();
+          break;
+        default:
+          throw new JsonDataException("Unexpected JSON token: " + reader.peek());
+      }
+    }
+  }
+
   @Override
   public List<DecodedTrace> getTraces() {
     return this.traces;

utils/test-agent-utils/decoder/src/main/java/datadog/trace/test/agent/decoder/json/raw/SpanJson.java

@@ -9,6 +9,7 @@
 import datadog.trace.test.agent.decoder.DecodedSpanLink;
 import datadog.trace.test.agent.decoder.DecodedSpanLinks;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -53,6 +54,12 @@ public final class SpanJson implements DecodedSpan {
   @Json(name = "span_links")
   List<SpanLinkJson> spanLinks;
 
+  // Only present in V1 payload. Kept as compact JSON text (number literals verbatim, see
+  // MessageJson.RawJsonAdapter) and exposed as the events meta tag by #resolveEvents
+  @Json(name = "span_events")
+  @MessageJson.RawJson
+  String spanEvents;
+
   // Decoded from span links meta tag or #spanLinks
   // transient so Moshi does not try to decode it
   transient List<DecodedSpanLink> links;
@@ -146,6 +153,21 @@ void resolveLinks() {
     this.links = unmodifiableList(decoded);
   }
 
+  /**
+   * Exposes the {@code span_events} field, when the agent filled it (the tracer submitted a {@code
+   * v1.0} payload), as the {@code events} meta tag, replacing any such tag. The value is the
+   * agent's JSON re-emitted compactly; its member order and attribute shape are the agent's, not
+   * those of the tag the tracer writes for older protocols.
+   */
+  void resolveEvents() {
+    if (this.spanEvents != null) {
+      if (this.meta == null) {
+        this.meta = new HashMap<>();
+      }
+      this.meta.put("events", this.spanEvents);
+    }
+  }
+
   @Override
   public String toString() {
     return "SpanJson{"