From 44e132cd1366427bfb9308815b966dd0314ae3cb Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 7 Oct 2026 08:39:28 -0400 Subject: [PATCH] Add the tag registry and map OpenTelemetry tag names through it Generates KnownTags (tag ids, Datadog and OpenTelemetry names, and the name<->id lookup) from tag-conventions.yaml at build time, and has TagMap canonicalize OpenTelemetry names to their Datadog tag so OTLP export can emit each tag under its OpenTelemetry name. Squashed from the review history of #12354. Co-Authored-By: Brice Dutheil Co-Authored-By: Claude Sonnet 5 Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Opus 5.5 --- .github/CODEOWNERS | 1 + build-logic/settings.gradle.kts | 7 +- build-logic/tag-registry/build.gradle.kts | 62 ++++ .../tagRegistry/GenerateKnownTagsTask.kt | 31 ++ .../tagRegistry/KnownTagsEmitter.kt | 221 +++++++++++ .../buildlogic/tagRegistry/TagConventions.kt | 346 ++++++++++++++++++ .../buildlogic/tagRegistry/TagRegistry.kt | 87 +++++ .../tagRegistry/TagRegistryExtension.kt | 25 ++ .../tagRegistry/TagRegistryGenerator.kt | 98 +++++ .../tagRegistry/TagRegistryGeneratorPlugin.kt | 29 ++ .../TagRegistryGeneratorPluginTest.kt | 278 ++++++++++++++ .../tagRegistry/TagRegistryGeneratorTest.kt | 305 +++++++++++++++ .../buildlogic/tagRegistry/TestFiles.kt | 11 + buildSrc/build.gradle.kts | 1 + .../OpenTelemetry14ConventionsTest.java | 13 +- .../datadog/trace/core/DDSpanContext.java | 2 + .../trace/core/otlp/trace/OtlpTraceJson.java | 43 ++- .../trace/core/otlp/trace/OtlpTraceProto.java | 43 ++- .../core/taginterceptor/TagInterceptor.java | 22 +- .../trace/OtlpTraceJsonCollectorTest.java | 61 +++ .../core/otlp/trace/OtlpTraceProtoTest.java | 136 ++++++- .../taginterceptor/TagInterceptorTest.java | 129 +++++++ gradle/java_no_deps.gradle | 2 +- internal-api/build.gradle.kts | 8 + .../java/datadog/trace/api/KnownTagCodec.java | 140 +++++++ .../main/java/datadog/trace/api/TagMap.java | 88 ++++- .../trace/api/EntryReadingHelperTest.java | 30 ++ .../java/datadog/trace/api/KnownTagsTest.java | 227 ++++++++++++ .../trace/api/TagMapNamespaceNamesTest.java | 81 ++++ tag-conventions.yaml | 169 +++++++++ 30 files changed, 2652 insertions(+), 44 deletions(-) create mode 100644 build-logic/tag-registry/build.gradle.kts create mode 100644 build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/GenerateKnownTagsTask.kt create mode 100644 build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt create mode 100644 build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt create mode 100644 build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt create mode 100644 build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryExtension.kt create mode 100644 build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt create mode 100644 build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPlugin.kt create mode 100644 build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPluginTest.kt create mode 100644 build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt create mode 100644 build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TestFiles.kt create mode 100644 internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java create mode 100644 internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java create mode 100644 internal-api/src/test/java/datadog/trace/api/TagMapNamespaceNamesTest.java create mode 100644 tag-conventions.yaml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ee8c267c77d..36f27ac1820 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,6 +12,7 @@ /README.md @DataDog/apm-java /SECURITY.md @DataDog/apm-java /SUPPORT.md @DataDog/apm-java +/tag-conventions.yaml @DataDog/apm-java @DataDog/apm-lang-platform-java @DataDog/apm-sdk-capabilities-java /dd-smoke-tests/README.md @DataDog/apm-java /docs/ @DataDog/apm-java diff --git a/build-logic/settings.gradle.kts b/build-logic/settings.gradle.kts index db1b1d5d1c8..a0dc4aa5b1e 100644 --- a/build-logic/settings.gradle.kts +++ b/build-logic/settings.gradle.kts @@ -55,6 +55,9 @@ dependencyResolutionManagement { rootProject.name = "build-logic" -include(":conventions") -include(":smoke-test") +include( + ":conventions", + ":smoke-test", + ":tag-registry", +) include(":testcontainers") diff --git a/build-logic/tag-registry/build.gradle.kts b/build-logic/tag-registry/build.gradle.kts new file mode 100644 index 00000000000..74029465ae5 --- /dev/null +++ b/build-logic/tag-registry/build.gradle.kts @@ -0,0 +1,62 @@ +plugins { + `java-gradle-plugin` + `kotlin-dsl` + `jvm-test-suite` + alias(libs.plugins.spotless) +} + +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} + +kotlin { + compilerOptions { + jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17) + } +} + +dependencies { + // Keep aligned with buildSrc, whose Jackson classes are visible to applied plugins. + implementation(platform("com.fasterxml.jackson:jackson-bom:2.17.2")) + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") +} + +gradlePlugin { + plugins { + create("tag-registry-generator") { + id = "dd-trace-java.tag-registry-generator" + implementationClass = "datadog.buildlogic.tagRegistry.TagRegistryGeneratorPlugin" + } + } +} + +testing { + suites { + named("test") { + useJUnitJupiter(libs.versions.junit5) + dependencies { + implementation(platform("org.junit:junit-bom:${libs.versions.junit5.get()}")) + implementation(libs.assertj.core) + implementation(libs.tabletest) + implementation(gradleTestKit()) + } + } + } +} + +spotless { + kotlin { + target("src/**/*.kt") + ktlint(libs.versions.ktlint.get()).editorConfigOverride( + mapOf( + "indent_size" to "2", + "ktlint_standard_trailing-comma-on-call-site" to "disabled", + "ktlint_standard_trailing-comma-on-declaration-site" to "disabled", + ), + ) + } + kotlinGradle { + ktlint(libs.versions.ktlint.get()).editorConfigOverride(mapOf("indent_size" to "2")) + } +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/GenerateKnownTagsTask.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/GenerateKnownTagsTask.kt new file mode 100644 index 00000000000..bd567a67854 --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/GenerateKnownTagsTask.kt @@ -0,0 +1,31 @@ +package datadog.buildlogic.tagRegistry + +import org.gradle.api.DefaultTask +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction + +/** + * Generates KnownTags.java and assignment reports from tag-conventions.yaml. + * Declares the inputs and outputs for Gradle's up-to-date checks and build cache. + */ +@CacheableTask +abstract class GenerateKnownTagsTask : DefaultTask() { + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + abstract val tagConventionsFile: RegularFileProperty + + @get:OutputDirectory abstract val destinationDirectory: DirectoryProperty + + @TaskAction + fun generate() { + val outDir = destinationDirectory.get().asFile + TagRegistryGenerator.generate(tagConventionsFile.get().asFile, outDir) + logger.lifecycle("tag-registry: generated -> $outDir") + } +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt new file mode 100644 index 00000000000..0b59976fc4d --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt @@ -0,0 +1,221 @@ +package datadog.buildlogic.tagRegistry + +import com.fasterxml.jackson.core.io.JsonStringEncoder +import java.util.Locale + +/** + * Emits the generated `KnownTags.java` from a [TagRegistry]. Public API first — per-tag + * `_NAME` (string) + `_ID` (encoded long, literal) couplets with a trailing `// makeTagId(...)` + * derivation comment — then the package-private `_SERIAL_NUM` constants, the + * `StringIndex.EmbeddingSupport` keyOf table and the resolver's name switches. + */ +object KnownTagsEmitter { + + fun emit(reg: TagRegistry, pkg: String, className: String): String { + // Sanitize a tag name into a Java identifier base (not yet unique). + fun sanitize(name: String): String { + var c = name.uppercase().replace(Regex("[^A-Za-z0-9]"), "_").replace(Regex("_+"), "_").trim('_') + if (c.isEmpty() || c[0].isDigit()) c = "T_$c" + return c + } + + // Collapse a duplicated trailing token so e.g. "resource.name" yields NAME (not NAME_NAME) and + // "_dd.parent_id" yields ID (not ID_ID); the non-duplicating pairs (ID + _NAME -> ID_NAME, NAME + // + _ID -> NAME_ID) are kept as-is. + fun withSuffix(base: String, suffix: String) = if (base.endsWith(suffix)) base else "$base$suffix" + + // NAME/ID/SERIAL_NUM constants are all fields of the same generated class, so uniqueness must be + // enforced on these FINAL suffixed identifiers, not on the pre-suffix base: two different base + // names can collapse to the same final identifier once a suffix is appended -- e.g. base + // "RESOURCE" suffixed with "_NAME" collides with a base that is already "RESOURCE_NAME" (which + // withSuffix leaves untouched, since it already ends with "_NAME"). + val used = HashSet() + fun unique(name: String): String { + var u = name + var n = 2 + while (!used.add(u)) { + u = "${name}_${n++}" + } + return u + } + + val nameOfConst = HashMap() + val idOfConst = HashMap() + val serialOfConst = HashMap() + val otelNameOfConst = HashMap() + for (t in reg.tags) { + val base = sanitize(t.name) + nameOfConst[t.name] = unique(withSuffix(base, "_NAME")) + idOfConst[t.name] = unique(withSuffix(base, "_ID")) + serialOfConst[t.name] = unique(withSuffix(base, "_SERIAL_NUM")) + // Suffix the pre-suffix base (not nameC), same as the other three: suffixing an + // already-suffixed identifier would produce a redundant compound like NAME_OTEL_NAME. + if (t.otelName != null) otelNameOfConst[t.name] = unique(withSuffix(base, "_OTEL_NAME")) + } + fun nameC(name: String) = nameOfConst.getValue(name) + fun idC(name: String) = idOfConst.getValue(name) + fun serialC(name: String) = serialOfConst.getValue(name) + fun otelNameC(name: String) = otelNameOfConst.getValue(name) + + val order = reg.tags.map { it.name } // stable emit order + // canonical name -> OpenTelemetry name, for the reverse (openTelemetryNameOf) switch. + val otelName = reg.tags.mapNotNull { t -> t.otelName?.let { t.name to it } }.toMap() + return buildString { + // Public API first (name + encoded id couplets), so readers see the useful parts up top; the + // serial ids and keyOf/resolver machinery follow below. Derivation is in the trailing comment. + appendLine( + """ + package $pkg; + + import datadog.trace.util.StringIndex; + + // GENERATED by the tag-registry code generator (dd-trace-java.tag-registry-generator). + // DO NOT EDIT. Source: tag-conventions.yaml. + public final class $className { + + // ---- tags ---- + """.trimIndent() + ) + + for (t in reg.tags) { + appendLine( + """ + public static final String ${nameC(t.name)} = "${escape(t.name)}"; + public static final long ${idC(t.name)} = ${hex(t.id)}; + """.trimIndent() + ) + if (t.otelName != null) { + appendLine(" public static final String ${otelNameC(t.name)} = \"${escape(t.otelName)}\";") + } + append("// makeTagId(serial=${t.serial})") + if (t.traceLevel) append(" + trace-level") + if (t.otelName != null) append(" -> ${escape(t.otelName)}") + appendLine(" <${escape(t.required)}>") + appendLine() + } + + // Serial numbers (globalSerial per tag) — package-private, consumed by the resolver switch. + appendLine(" // ---- serial numbers ----") + for (t in reg.tags) { + appendLine(" static final int ${serialC(t.name)} = ${t.serial};") + } + + // OpenTelemetry name -> canonical tag name. Validation ensures aliases are distinct from all + // canonical names. Sort by OTel name to keep output deterministic. + val otelByCanonical = + reg.tags + .mapNotNull { t -> t.otelName?.let { it to t.name } } + .sortedBy { it.first } + + // keyOf table (open-addressed, via StringIndex.EmbeddingSupport). Canonical names first, then + // OpenTelemetry names -- an OTel name resolves to its canonical tag's id (there is no distinct id + // for it), so keyOf(otelName) == keyOf(canonical); nameOf still returns the canonical name. + appendLine( + """ + + private static final String[] KEYOF_NAMES = { + """.trimIndent() + ) + order.forEach { appendLine(" ${nameC(it)},") } + otelByCanonical.forEach { (otel, _) -> + appendLine(" \"${escape(otel)}\",") + } + appendLine( + """ + }; + private static final long[] KEYOF_VALUES = { + """.trimIndent() + ) + order.forEach { appendLine(" ${idC(it)},") } + otelByCanonical.forEach { (_, canonical) -> + appendLine(" ${idC(canonical)},") + } + // Resolver. KnownTagCodec.Installed links to this field directly, so merely resolving a tag + // name initializes this class -- there is no registration step and no ordering to get wrong. + appendLine( + """ + }; + private static final int[] KEYOF_HASHES; + private static final String[] KEYOF_KEYS; + private static final long[] KEYOF_IDS; + + static { + StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES); + long[] ids = new long[data.names.length]; + for (int j = 0; j < KEYOF_NAMES.length; j++) { + ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = + KEYOF_VALUES[j]; + } + KEYOF_HASHES = data.hashes; + KEYOF_KEYS = data.names; + KEYOF_IDS = ids; + } + + /** + * The registry's name↔id tables, as a {@link KnownTagCodec.Resolver}. {@code KnownTagCodec} + * reads this field from its own holder, so the two classes complete each other: the codec owns + * the bit layout and the naming policy, this class owns the data. Nothing has to be called first. + */ + static final KnownTagCodec.Resolver RESOLVER = + new KnownTagCodec.Resolver() { + @Override + public String nameOf(long tagId) { + switch (KnownTagCodec.serialNum(tagId)) { + """.trimIndent() + ) + for (name in order) { + appendLine( + """ + case ${serialC(name)}: + return ${nameC(name)}; + """.trimIndent() + ) + } + // openTelemetryNameOf: canonical id -> OTel-namespace name, null when the tag has none. The + // caller (a serializer) owns any fall-back-to-Datadog-name policy; this stays a pure lookup. + appendLine( + """ + default: + return null; + } + } + + @Override + public String openTelemetryNameOf(long tagId) { + switch (KnownTagCodec.serialNum(tagId)) { + """.trimIndent() + ) + for (name in order) { + if (otelName[name] == null) continue + appendLine( + """ + case ${serialC(name)}: + return ${otelNameC(name)}; + """.trimIndent() + ) + } + appendLine( + """ + default: + return null; + } + } + + @Override + public long keyOf(String name) { + int slot = StringIndex.EmbeddingSupport.indexOf(KEYOF_HASHES, KEYOF_KEYS, name); + return slot < 0 ? 0L : KEYOF_IDS[slot]; + } + }; + + private $className() {} + } + """.trimIndent() + ) + } + } + + private fun hex(id: Long): String = "0x%016XL".format(Locale.ROOT, id) + + private fun escape(value: String): String = String(JsonStringEncoder.getInstance().quoteAsString(value)) +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt new file mode 100644 index 00000000000..75eaba1fe70 --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt @@ -0,0 +1,346 @@ +package datadog.buildlogic.tagRegistry + +/** + * Parses the tag conventions and resolves each span type's tags through `extends`, `include`, + * and `applies`. Declarations carry names, types, requirement levels, and optional OpenTelemetry + * renames. ID assignment and source generation are handled separately. + */ +class TagConventions private constructor( + private val spanTypes: Map, + private val mixins: Map, + private val traceLevel: List, +) { + /** One tag declaration: its Datadog name, type, requirement level, and optional rename. */ + data class Tag( + val name: String, + val type: String, + val required: String, + /** + * Explicit OpenTelemetry name from `otel-name`, or null when no rename is configured. + * When a rename is configured, both names resolve to the same tag ID. + * + * An omitted `otel-name` and the reserved literal `none` both produce null. Exporters + * currently use the Datadog name in either case; `none` does not suppress the tag yet. + */ + val otelName: String? = null, + /** + * Set `span-kind-neutral: true` for a rename declared on a concrete span type only + * when the Datadog and OpenTelemetry names denote the same value on every span kind. + * Name resolution ignores span kind, so the rename also applies outside that type. + * + * For example, `db.type` -> `db.system` on `db.client` qualifies: `db.system` only + * ever describes a database. Renames in `trace_level`, abstract types, and mixins need + * no flag. Setting this flag without a configured rename is invalid. + * + * The flag is interim: a follow-on replaces it with span-kind-aware name resolution. + */ + val spanKindNeutral: Boolean = false, + ) + + /** + * A `{ ref: , required: }` entry. It reuses a tag declared elsewhere, optionally + * at a different requirement level; its type and otel-name always come from that declaration. + */ + data class Ref(val name: String, val required: String?) + + data class SpanType( + val name: String, + val abstract: Boolean, + val extends: String?, + val include: List, + val tags: List, + val refs: List = emptyList(), + ) + + data class Mixin( + val name: String, + val appliesAll: Boolean, + val appliesTo: Set, + val tags: List, + val refs: List = emptyList(), + ) + + private val declarations: Map by lazy { + allDeclaredTags().associateBy { it.name } + } + + /** The tag a [Ref] names, at the ref's requirement level when it overrides one. */ + private fun materialize(ref: Ref): Tag { + val decl = declarations.getValue(ref.name) + return if (ref.required == null) decl else decl.copy(required = ref.required) + } + + /** Returns the names of non-abstract span types in sorted order. */ + fun concreteTypes(): List = spanTypes.values.filter { !it.abstract }.map { it.name }.sorted() + + /** + * Resolves tags from the root ancestor to the requested type, then adds mixins whose + * `applies` matches the type or an ancestor. + * + * Each type contributes its declarations, included mixins, then references. Duplicate + * names keep their first position; references can override requirement levels. References + * from `applies` mixins add only missing tags. + */ + fun resolve(typeName: String): List { + val result = LinkedHashMap() + fun add(t: Tag) = result.putIfAbsent(t.name, t) + fun applyRef(r: Ref) { + val current = result[r.name] + // Re-putting an existing key keeps its LinkedHashMap position. + result[r.name] = + when { + current == null -> materialize(r) + r.required == null -> current + else -> current.copy(required = r.required) + } + } + + val chain = ArrayList() + var cur: SpanType? = spanTypes[typeName] + while (cur != null) { + chain.add(cur) + cur = cur.extends?.let { spanTypes[it] } + } + for (st in chain.asReversed()) { + st.tags.forEach { add(it) } + for (mixinName in st.include) { + mixins[mixinName]?.let { mx -> + mx.tags.forEach { add(it) } + mx.refs.forEach { applyRef(it) } + } + } + st.refs.forEach { applyRef(it) } + } + val chainNames = chain.map { it.name }.toSet() + for (mx in mixins.values) { + if (mx.appliesAll || mx.appliesTo.any { it in chainNames }) { + mx.tags.forEach { add(it) } + mx.refs.forEach { if (it.name !in result) add(materialize(it)) } + } + } + return result.values.toList() + } + + /** Returns the tags declared in `trace_level`, which are set once per trace, not per span. */ + fun traceLevelTags(): List = traceLevel + + /** Includes every declaration, even from mixins whose span types are not modeled yet. */ + fun allDeclaredTags(): List = buildList { + addAll(traceLevel) + spanTypes.toSortedMap().values.forEach { addAll(it.tags) } + mixins.toSortedMap().values.forEach { addAll(it.tags) } + }.distinctBy { it.name } + + /** + * Returns mixins with `applies` targets missing from `span_types`, paired with the missing names. + * + * These targets are reported in `resolved-tags.txt` without failing generation. The mixin's + * tags are still registered, and any matching declared span types still receive them. + */ + fun unmodeledAppliesTargets(): List>> = mixins.values + .sortedBy { it.name } + .mapNotNull { mx -> + val missing = mx.appliesTo.filter { it !in spanTypes }.sorted() + if (missing.isEmpty()) null else mx.name to missing + } + + companion object { + @Suppress("UNCHECKED_CAST") + fun parse(root: Map): TagConventions { + for (section in listOf("span_types", "mixins", "trace_level")) { + require(root[section] == null || root[section] is Map<*, *>) { "$section must be a mapping" } + } + val spanTypesRaw = (root["span_types"] as? Map) ?: emptyMap() + val spanTypes = + spanTypesRaw.mapValues { (name, v) -> + require(v is Map<*, *>) { "span type '$name' must be a mapping" } + val m = v as Map + require(m["abstract"] == null || m["abstract"] is Boolean) { + "span type '$name' abstract must be a boolean" + } + require(m["extends"] == null || m["extends"] is String) { + "span type '$name' extends must be a span type name" + } + val include = m["include"] + require(include == null || (include is List<*> && include.all { it is String })) { + "span type '$name' include must be a list of mixin names" + } + SpanType( + name = name, + abstract = (m["abstract"] as? Boolean) ?: false, + extends = m["extends"] as? String, + include = (m["include"] as? List) ?: emptyList(), + tags = tagList(m["tags"]), + refs = refList(m["tags"]), + ) + } + + val mixinsRaw = (root["mixins"] as? Map) ?: emptyMap() + val mixins = + mixinsRaw.mapValues { (name, v) -> + require(v is Map<*, *>) { "mixin '$name' must be a mapping" } + val m = v as Map + val applies = m["applies"] + require(applies == null || applies == "all" || (applies is List<*> && applies.all { it is String })) { + "mixin '$name' applies must be 'all' or a list of span types" + } + Mixin( + name = name, + appliesAll = applies == "all", + appliesTo = if (applies is List<*>) (applies as List).toSet() else emptySet(), + tags = tagList(m["tags"]), + refs = refList(m["tags"]), + ) + } + + for (spanType in spanTypes.values) { + for (included in spanType.include) { + require(included in mixins) { + "span type '${spanType.name}' includes unknown mixin '$included'" + } + } + val visited = HashSet() + var current: SpanType? = spanType + while (current != null) { + val name = current.name + require(visited.add(name)) { "span type '${spanType.name}' has cyclic extends at '$name'" } + current = current.extends?.let { parent -> + requireNotNull(spanTypes[parent]) { "span type '$name' extends unknown span type '$parent'" } + } + } + } + + // Trace-level tags pass through under their Datadog name for now; their OTel mapping (resource + // attributes) is a follow-on. TODO(otel follow-on). + val traceLevelRaw = (root["trace_level"] as? Map)?.get("tags") + val traceLevel = tagList(traceLevelRaw) + require(refList(traceLevelRaw).isEmpty()) { "trace_level tags must be declarations, not refs" } + validateSingleDeclaration(spanTypes, mixins, traceLevel) + validateOtelNameScope(spanTypes, mixins, traceLevel) + return TagConventions(spanTypes, mixins, traceLevel) + } + + /** + * Rejects multiple declarations of the same `dd-name`, including identical declarations. + * Declare shared tags once on a parent or mixin and reuse them through `ref` entries, + * which may override only `required`. References must name a declared tag. + */ + private fun validateSingleDeclaration( + spanTypes: Map, + mixins: Map, + traceLevel: List, + ) { + val home = HashMap() // name -> declaring container + val declare = { container: String, t: Tag -> + val prev = home.putIfAbsent(t.name, container) + require(prev == null) { + "tag '${t.name}' is declared in both '$prev' and '$container'. Declare it once and use " + + "`{ ref: ${t.name}, required: }` elsewhere; a ref may override only `required`." + } + } + traceLevel.forEach { declare("", it) } + spanTypes.values.forEach { st -> st.tags.forEach { declare(st.name, it) } } + mixins.values.forEach { mx -> mx.tags.forEach { declare("mixin ${mx.name}", it) } } + + val refs = + spanTypes.values.flatMap { st -> st.refs.map { st.name to it } } + + mixins.values.flatMap { mx -> mx.refs.map { "mixin ${mx.name}" to it } } + for ((container, r) in refs) { + require(r.name in home) { "'$container' refs undeclared tag '${r.name}'" } + } + } + + /** + * Requires `span-kind-neutral: true` for renames on concrete span types because name + * resolution ignores span kind. Validation checks the flag and requires a configured + * rename, relying on the author's semantic check. Shared scopes do not require the flag. + */ + private fun validateOtelNameScope( + spanTypes: Map, + mixins: Map, + traceLevel: List, + ) { + val all = traceLevel + spanTypes.values.flatMap { it.tags } + mixins.values.flatMap { it.tags } + for (t in all) { + require(!t.spanKindNeutral || t.otelName != null) { + "tag '${t.name}' sets span-kind-neutral without an otel-name" + } + } + for (st in spanTypes.values.filter { !it.abstract }) { + for (t in st.tags) { + require(t.otelName == null || t.spanKindNeutral) { + "tag '${t.name}' renames to otel-name '${t.otelName}' on concrete span type '${st.name}'. " + + "Canonicalization ignores span kind, so either declare it in a shared scope (an " + + "abstract parent or a mixin) or, if '${t.otelName}' means '${t.name}' on every span " + + "kind, add `span-kind-neutral: true`." + } + } + } + } + + private val REF_KEYS = setOf("ref", "required") + + @Suppress("UNCHECKED_CAST") + private fun refList(tags: Any?): List = (tags as? List>) + ?.filter { it.containsKey("ref") } + ?.map { m -> + val extra = m.keys - REF_KEYS + require(extra.isEmpty()) { + "ref '${m["ref"]}' may override only `required`, but also sets $extra; identity " + + "(type, otel-name) comes from the tag's single declaration" + } + val name = m["ref"] + require(name is String && name.isNotBlank()) { "ref has no valid tag name: $m" } + require(m["required"] == null || m["required"] is String) { "ref '$name' required must be a string" } + Ref(name, m["required"] as? String) + } ?: emptyList() + + @Suppress("UNCHECKED_CAST") + private fun tagList(tags: Any?): List { + require(tags == null || tags is List<*>) { "tags must be a list of tag declarations" } + return (tags as? List<*>)?.mapNotNull { entry -> + require(entry is Map<*, *>) { "tag declaration must be a mapping" } + val m = entry as Map + if (m.containsKey("ref")) return@mapNotNull null + val name = parseDdName(m) + require(m["type"] == null || m["type"] is String) { "tag '$name' type must be a string" } + require(m["required"] == null || m["required"] is String) { "tag '$name' required must be a string" } + Tag( + name = name, + type = (m["type"] as? String) ?: "string", + required = (m["required"] as? String) ?: "optional", + otelName = parseOtelName(m), + spanKindNeutral = parseSpanKindNeutral(m), + ) + } ?: emptyList() + } + + /** Reads the required, nonblank string `dd-name`; invalid values fail generation. */ + private fun parseDdName(m: Map): String { + val raw = m["dd-name"] + require(raw is String && raw.isNotBlank()) { "tag declaration has no valid dd-name: $m" } + return raw + } + + private fun parseSpanKindNeutral(m: Map): Boolean { + if (!m.containsKey("span-kind-neutral")) return false + val raw = m["span-kind-neutral"] + require(raw is Boolean) { "tag '${m["dd-name"]}' has a non-boolean span-kind-neutral: '$raw'" } + return raw + } + + /** + * Reads `otel-name`: an omitted key or `none` returns null; otherwise returns a nonblank string. + * Rejects explicit null, blank strings, and non-string values so typos cannot disable a rename. + */ + private fun parseOtelName(m: Map): String? { + if (!m.containsKey("otel-name")) return null // absent => pass-through + val raw = m["otel-name"] + require(raw is String && raw.isNotBlank()) { + "tag '${m["dd-name"]}' has an invalid otel-name: '$raw'. Use a non-empty name, the literal " + + "`none`, or omit the key entirely for pass-through under the Datadog name." + } + return raw.takeUnless { it == "none" } + } + } +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt new file mode 100644 index 00000000000..5f12adee3b9 --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt @@ -0,0 +1,87 @@ +package datadog.buildlogic.tagRegistry + +/** + * Assigns tag ids from a parsed [TagConventions]. The id encoding mirrors KnownTagCodec: [63-48 + * serial][47-32 reserved][31-0 flags]. + * + *

An id is IDENTITY only: a globally unique serial plus the trace-level classification bit. It + * carries no storage-layout coordinate -- bits [47-32] are held vacant for the co-occurrence slot + * that the dense tag store assigns by graph coloring, which lands with the dense store itself. + * Nothing here needs to know how (or whether) a tag is stored. + * + *

Nor does anything here know how a tag is SET. Whether the tracer intercepts a tag on the + * set-path (routing it to a span field or a sampling directive instead of tag storage) is a + * property of TagInterceptor, not of the tag's identity, and modelling it was the source of a whole + * class of drift between this registry and the interceptor's actual switch. It arrives with the + * work that consumes it -- the id->handler dispatch table that retires TagInterceptor -- where the + * interceptor can be the authority. Re-adding a classification bit then is purely additive. + */ +class TagRegistry private constructor(val tags: List) { + data class Tag( + val name: String, + val type: String, + val required: String, + val serial: Int, + val traceLevel: Boolean, + val id: Long, + val otelName: String? = null, + ) + + companion object { + const val FIRST_SERIAL = 1 + const val LEVEL_TRACE = 1L shl 2 // low-32 carve bit 2; mirrors KnownTagCodec.LEVEL_TRACE + + /** + * Mirrors KnownTagCodec.makeTagId(serial) + traceLevel() -- must stay in sync. LEVEL_TRACE at + * bit 2, other low bits and the reserved [47-32] window zero. + */ + fun encode(serial: Int, traceLevel: Boolean): Long { + var id = serial.toLong() shl 48 + if (traceLevel) id = id or LEVEL_TRACE + return id + } + + fun build(conv: TagConventions): TagRegistry { + val traceNames = conv.traceLevelTags().map { it.name }.toSet() + + // Stable order (by name) so serials -- and therefore ids -- are a pure function of the input. + val tags = + conv.allDeclaredTags().sortedBy { it.name }.mapIndexed { i, t -> + val serial = FIRST_SERIAL + i + val traceLevel = t.name in traceNames + Tag( + t.name, + t.type, + t.required, + serial, + traceLevel, + id = encode(serial, traceLevel), + otelName = t.otelName + ) + } + + validateOtelNames(tags) + return TagRegistry(tags) + } + + /** + * An OpenTelemetry name must be unambiguous: it may not collide with any canonical tag name, nor + * be claimed by two different tags. Otherwise keyOf(otelName) would have no single right answer. + * Fail the build loudly rather than silently pick a winner. + */ + private fun validateOtelNames(tags: List) { + val canonical = tags.map { it.name }.toSet() + val owner = HashMap() + for (t in tags) { + val otel = t.otelName ?: continue + require(otel !in canonical) { + "OpenTelemetry name '$otel' (of '${t.name}') collides with canonical tag name '$otel'" + } + val prev = owner.put(otel, t.name) + require(prev == null) { + "OpenTelemetry name '$otel' is claimed by both '$prev' and '${t.name}'" + } + } + } + } +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryExtension.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryExtension.kt new file mode 100644 index 00000000000..f6d07ca586b --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryExtension.kt @@ -0,0 +1,25 @@ +package datadog.buildlogic.tagRegistry + +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.ProjectLayout +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory +import javax.inject.Inject + +/** Extension configuring the tag-registry generator inputs/outputs. */ +abstract class TagRegistryExtension @Inject constructor( + objectFactory: ObjectFactory, + layout: ProjectLayout +) { + /** + * Define the location of the tag conventions + */ + abstract val tagConventionsFile: RegularFileProperty + + /** + * Destination of the generated sources, by convention under `build/generated/tag-registry`. + */ + val destinationDirectory: DirectoryProperty = objectFactory.directoryProperty().convention( + layout.buildDirectory.dir("generated/tag-registry") + ) +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt new file mode 100644 index 00000000000..3d2885b3113 --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt @@ -0,0 +1,98 @@ +package datadog.buildlogic.tagRegistry + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import java.io.File +import java.util.Locale + +/** Emits the Java tag registry and reports in a deterministic order. */ +object TagRegistryGenerator { + /** Parses the conventions YAML and writes the full generated tree under [outDir]. */ + fun generate(tagConventionsFile: File, outDir: File) { + val mapper = ObjectMapper(YAMLFactory()) + val domain: Map = + tagConventionsFile.inputStream().use { + mapper.readValue(it, object : TypeReference>() {}) + } + + // Validate before touching the destination tree: an invalid domain model must fail loudly, + // not after the previous (valid) generated output has already been wiped out. + val conv = TagConventions.parse(domain) + val reg = TagRegistry.build(conv) + + // Remove obsolete generated files when the output changes. + outDir.deleteRecursively() + outDir.mkdirs() + // KnownTags.java goes under java/ (added as a srcDir); the .txt reports sit at the root. + val javaPkg = File(outDir, "java/datadog/trace/api").apply { mkdirs() } + + File(outDir, "resolved-tags.txt").writeText(resolvedReport(conv)) + File(outDir, "tag-assignment.txt").writeText(assignmentReport(reg)) + File(javaPkg, "KnownTags.java") + .writeText(KnownTagsEmitter.emit(reg, "datadog.trace.api", "KnownTags")) + } + + /** resolved-tags.txt — the per-type resolved sets (composition check). */ + private fun resolvedReport(conv: TagConventions) = buildString { + appendLine("# Resolved per-type tag sets (concrete span types).") + val unmodeled = conv.unmodeledAppliesTargets() + if (unmodeled.isNotEmpty()) { + appendLine( + """ + # + # LAYOUT GAP: these mixins apply to span types not modeled here, so they + # contribute to no resolved set below. Their tags ARE registered (an id is + # identity, not layout) -- they simply occupy no per-type slot yet. + """.trimIndent() + ) + for ((mixin, missing) in unmodeled) { + appendLine("# $mixin -> ${missing.joinToString(", ")}") + } + } + for (type in conv.concreteTypes()) { + val tags = conv.resolve(type) + appendLine( + """ + + $type (${tags.size} tags): + """.trimIndent() + ) + for (t in tags) appendLine(" - ${t.name}") + } + } + + /** tag-assignment.txt — serials, ids, and the OpenTelemetry name mapping (identity check). */ + private fun assignmentReport(reg: TagRegistry) = buildString { + appendLine( + """ + # Tag id assignment. tags=${reg.tags.size} + + # TAGS serial lvl id required name + """.trimIndent() + ) + for ((name, _, required, serial, traceLevel, id) in reg.tags) { + appendLine( + " %6d %s %-18s %-12s %s".format( + Locale.ROOT, + serial, + if (traceLevel) "T" else "-", + "0x%016X".format(Locale.ROOT, id), + required, + name + ) + ) + } + appendLine( + """ + + # OPENTELEMETRY NAMES. keyOf(otelName) resolves to the canonical tag's id; nameOf still + # returns the Datadog name, openTelemetryNameOf returns the name below. (No distinct id.) + """.trimIndent() + ) + val otelPairs = reg.tags.mapNotNull { t -> t.otelName?.let { it to t.name } }.sortedBy { it.first } + for ((otel, canonical) in otelPairs) { + appendLine(" %-30s -> %s".format(Locale.ROOT, otel, canonical)) + } + } +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPlugin.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPlugin.kt new file mode 100644 index 00000000000..c350d3c302e --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPlugin.kt @@ -0,0 +1,29 @@ +package datadog.buildlogic.tagRegistry + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.SourceSetContainer +import org.gradle.kotlin.dsl.configure +import org.gradle.kotlin.dsl.create +import org.gradle.kotlin.dsl.register + +/** Generates the tag registry and adds its Java output to the main source set. */ +class TagRegistryGeneratorPlugin : Plugin { + override fun apply(project: Project) { + val ext = project.extensions.create("tagRegistry") + val generate = + project.tasks.register("generateKnownTags") { + group = "build" + description = "Generates the Java tag registry and assignment reports." + tagConventionsFile.convention(ext.tagConventionsFile) + destinationDirectory.convention(ext.destinationDirectory) + } + project.pluginManager.withPlugin("java") { + project.extensions.configure { + named("main") { + java.srcDir(generate.flatMap { it.destinationDirectory.dir("java") }) + } + } + } + } +} diff --git a/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPluginTest.kt b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPluginTest.kt new file mode 100644 index 00000000000..fa08bc5b177 --- /dev/null +++ b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPluginTest.kt @@ -0,0 +1,278 @@ +package datadog.buildlogic.tagRegistry + +import org.assertj.core.api.Assertions.assertThat +import org.gradle.testkit.runner.GradleRunner +import org.gradle.testkit.runner.TaskOutcome.FROM_CACHE +import org.gradle.testkit.runner.TaskOutcome.SUCCESS +import org.gradle.testkit.runner.TaskOutcome.UP_TO_DATE +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import org.tabletest.junit.TableTest +import java.io.File +import java.net.URLClassLoader +import java.util.jar.JarFile + +class TagRegistryGeneratorPluginTest { + @TempDir lateinit var directory: File + + @TableTest( + """ + scenario | plugins + Java first | 'java; id("dd-trace-java.tag-registry-generator")' + tag registry first | 'id("dd-trace-java.tag-registry-generator"); java' + """ + ) + fun `Java compilation and source consumers generate the registry in either plugin order`( + plugins: String + ) { + gradleProject( + """ + plugins { $plugins } + java { withSourcesJar() } + """ + ) + writeJavaDependencies() + + val result = runner("compileJava", "sourcesJar", "javadoc").build() + + assertThat(result.task(":generateKnownTags")?.outcome).isEqualTo(SUCCESS) + assertThat(result.task(":compileJava")?.outcome).isEqualTo(SUCCESS) + assertThat(File(directory, "build/classes/java/main/datadog/trace/api/KnownTags.class")).exists() + assertGeneratedFiles() + assertThat(File(directory, "src/generated")).doesNotExist() + JarFile(File(directory, "build/libs/fixture-sources.jar")).use { jar -> + assertThat(jar.getEntry("datadog/trace/api/KnownTags.java")).isNotNull() + } + assertThat(File(directory, "build/docs/javadoc/datadog/trace/api/KnownTags.html")).exists() + } + + @Test + fun `tag names containing Java special characters compile and retain their values`() { + gradleProject( + """ + plugins { + java + id("dd-trace-java.tag-registry-generator") + } + """ + ) + directory.conventionsFile( + """ + span_types: + base: + tags: + - dd-name: 'tag"\name' + otel-name: "alias\"\\name\nline" + span-kind-neutral: true + """ + ) + writeJavaDependencies() + + val result = runner("compileJava").build() + + assertThat(result.task(":compileJava")?.outcome).isEqualTo(SUCCESS) + assertGeneratedFiles() + val classes = File(directory, "build/classes/java/main").toURI().toURL() + URLClassLoader(arrayOf(classes), null).use { loader -> + val names = loader.loadClass("datadog.trace.api.KnownTags").fields + .filter { it.type == String::class.java }.map { it.get(null) } + assertThat(names).containsExactlyInAnyOrder("tag\"\\name", "alias\"\\name\nline") + } + } + + @Test + fun `compilation reuses the configuration cache and regenerates after YAML changes`() { + gradleProject( + """ + plugins { + java + id("dd-trace-java.tag-registry-generator") + } + """ + ) + writeJavaDependencies() + assertThat(runner("compileJava").build().task(":generateKnownTags")?.outcome) + .isEqualTo(SUCCESS) + + val second = runner("compileJava").build() + assertThat(second.output).contains("Reusing configuration cache.") + assertThat(second.task(":generateKnownTags")?.outcome).isEqualTo(UP_TO_DATE) + + directory.conventionsFile( + """ + span_types: + base: + tags: [{dd-name: updated}] + """ + ) + val third = runner("compileJava").build() + assertThat(third.output).contains("Reusing configuration cache.") + assertThat(third.task(":generateKnownTags")?.outcome).isEqualTo(SUCCESS) + assertThat(third.task(":compileJava")?.outcome).isEqualTo(SUCCESS) + assertGeneratedFiles() + assertThat(generated().readText()).contains("UPDATED_NAME").doesNotContain("FOO_NAME") + } + + @Test + fun `generation can be restored from cache in a different project directory`() { + val firstProject = File(directory, "first") + val secondProject = File(directory, "second") + gradleProject(projectDir = firstProject) + gradleProject(projectDir = secondProject) + val cache = File(directory, "cache").toURI() + val settings = """ + rootProject.name = "fixture" + buildCache { local { directory = uri("$cache") } } + """ + firstProject.writeFile("settings.gradle.kts", settings) + secondProject.writeFile("settings.gradle.kts", settings) + + assertThat( + runner("generateKnownTags", "--build-cache", projectDir = firstProject) + .build().task(":generateKnownTags")?.outcome + ).isEqualTo(SUCCESS) + val result = runner("generateKnownTags", "--build-cache", projectDir = secondProject).build() + + assertThat(result.task(":generateKnownTags")?.outcome).isEqualTo(FROM_CACHE) + assertGeneratedFiles(File(firstProject, "build/generated/tag-registry")) + assertGeneratedFiles(File(secondProject, "build/generated/tag-registry")) + assertThat(generated(secondProject).readText()).isEqualTo(generated(firstProject).readText()) + } + + @Test + fun `output follows a custom build directory and clean removes it`() { + gradleProject( + """ + plugins { + id("dd-trace-java.tag-registry-generator") + java + } + layout.buildDirectory.set(layout.projectDirectory.dir("custom-build")) + """ + ) + writeJavaDependencies() + + val result = runner("compileJava").build() + + assertThat(result.task(":generateKnownTags")?.outcome).isEqualTo(SUCCESS) + val output = File(directory, "custom-build/generated/tag-registry") + assertGeneratedFiles(output) + assertThat(File(directory, "build/generated")).doesNotExist() + runner("clean").build() + assertThat(output).doesNotExist() + } + + @Test + fun `input and output can be overridden without losing the producer dependency`() { + gradleProject( + """ + plugins { + java + id("dd-trace-java.tag-registry-generator") + } + tagRegistry { + tagConventionsFile.set(layout.projectDirectory.file("custom.yaml")) + destinationDirectory.set(layout.buildDirectory.dir("custom-generated")) + } + """ + ) + File(directory, "tag-conventions.yaml").renameTo(File(directory, "custom.yaml")) + writeJavaDependencies() + + val result = runner("compileJava").build() + + assertThat(result.task(":generateKnownTags")?.outcome).isEqualTo(SUCCESS) + assertGeneratedFiles(File(directory, "build/custom-generated")) + assertThat(File(directory, "build/classes/java/main/datadog/trace/api/KnownTags.class")).exists() + } + + @Test + fun `help does not read or generate the domain model`() { + gradleProject() + File(directory, "tag-conventions.yaml").delete() + + val result = runner("help").build() + + assertThat(result.task(":generateKnownTags")).isNull() + assertThat(generated()).doesNotExist() + } + + private fun gradleProject( + build: String = """ + plugins { id("dd-trace-java.tag-registry-generator") } + """, + projectDir: File = directory + ) { + projectDir.writeFile( + "settings.gradle.kts", + """ + rootProject.name = "fixture" + """ + ) + projectDir.writeFile( + "build.gradle.kts", + build, + """ + tagRegistry { + tagConventionsFile.convention(layout.projectDirectory.file("tag-conventions.yaml")) + } + """ + ) + projectDir.conventionsFile( + """ + span_types: + base: + tags: [{dd-name: foo}, {dd-name: foo.name}] + """ + ) + } + + private fun runner(vararg arguments: String, projectDir: File = directory): GradleRunner = GradleRunner.create().withProjectDir(projectDir).withPluginClasspath() + .withArguments(*arguments, "--configuration-cache", "--stacktrace") + + private fun generated(projectDir: File = directory): File = File(projectDir, "build/generated/tag-registry/java/datadog/trace/api/KnownTags.java") + + private fun assertGeneratedFiles(output: File = File(directory, "build/generated/tag-registry")) { + assertThat(File(output, "java/datadog/trace/api/KnownTags.java")).isFile() + assertThat(File(output, "resolved-tags.txt")).isFile() + assertThat(File(output, "tag-assignment.txt")).isFile() + } + + // Only the API surface is needed here; internal-api tests exercise the real resolver at runtime. + private fun writeJavaDependencies() { + directory.writeFile( + "src/main/java/datadog/trace/api/KnownTagCodec.java", + """ + package datadog.trace.api; + public final class KnownTagCodec { + public interface Resolver { + String nameOf(long id); + String openTelemetryNameOf(long id); + long keyOf(String name); + } + public static int serialNum(long id) { return (int) (id >>> 48); } + } + """ + ) + directory.writeFile( + "src/main/java/datadog/trace/util/StringIndex.java", + """ + package datadog.trace.util; + public final class StringIndex { + public static final class Data { + public final int[] hashes; + public final String[] names; + public Data(String[] names) { this.names = names; this.hashes = new int[names.length]; } + } + public static final class EmbeddingSupport { + public static Data create(String[] names) { return new Data(names); } + public static int indexOf(int[] hashes, String[] names, String name) { + for (int i = 0; i < names.length; i++) if (names[i].equals(name)) return i; + return -1; + } + } + } + """ + ) + } +} diff --git a/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt new file mode 100644 index 00000000000..e05b4ce75a6 --- /dev/null +++ b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt @@ -0,0 +1,305 @@ +package datadog.buildlogic.tagRegistry + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatIllegalArgumentException +import org.intellij.lang.annotations.Language +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import org.tabletest.junit.TableTest +import java.io.File + +class TagRegistryGeneratorTest { + @TempDir + lateinit var directory: File + + @Test + fun `generation is deterministic and removes obsolete files`() { + val yaml = directory.conventionsFile( + """ + trace_level: + tags: [{dd-name: env}] + span_types: + base: + abstract: true + tags: [{dd-name: service, otel-name: service.name}] + http.server: + extends: base + include: [peer] + tags: [{dd-name: http.method, otel-name: http.request.method, span-kind-neutral: true}] + mixins: + common: + applies: all + tags: [{dd-name: span.kind}] + peer: + tags: [{dd-name: peer.service}] + ci_visibility: + applies: [test] + tags: [{dd-name: test.name}] + """ + ) + val output = File(directory, "generated") + TagRegistryGenerator.generate(yaml, output) + val first = contents(output) + output.writeFile("obsolete.txt", "obsolete") + + TagRegistryGenerator.generate(yaml, output) + + assertThat(contents(output)).isEqualTo(first) + assertThat(first.keys).containsExactlyInAnyOrder( + "java/datadog/trace/api/KnownTags.java", + "resolved-tags.txt", + "tag-assignment.txt" + ) + assertThat(first.getValue("resolved-tags.txt")) + .contains("http.server (4 tags):", " - service", " - peer.service", " - span.kind", "ci_visibility -> test") + assertThat(first.getValue("tag-assignment.txt")) + .contains("tags=6", "test.name", "service.name") + assertThat(first.getValue("java/datadog/trace/api/KnownTags.java")) + .contains("ENV_ID = 0x0001000000000004L", "TEST_NAME", "http.request.method") + } + + @TableTest( + """ + scenario | tag | message + missing Datadog name | '{type: string}' | no valid dd-name + empty Datadog name | "{dd-name: ''}" | no valid dd-name + nonstring Datadog name | '{dd-name: 42}' | no valid dd-name + null OpenTelemetry name | '{dd-name: foo, otel-name: null}' | invalid otel-name + empty OpenTelemetry name | "{dd-name: foo, otel-name: ''}" | invalid otel-name + nonstring type | '{dd-name: foo, type: 42}' | type must be a string + nonstring required | '{dd-name: foo, required: 42}' | required must be a string + """ + ) + fun `invalid declarations preserve previous output`(tag: String, message: String) { + val yaml = directory.conventionsFile("span_types: {base: {tags: [{dd-name: valid}]}}") + val output = File(directory, "generated") + TagRegistryGenerator.generate(yaml, output) + val previous = contents(output) + directory.conventionsFile("span_types: {base: {tags: [$tag]}}") + + assertThatIllegalArgumentException().isThrownBy { TagRegistryGenerator.generate(yaml, output) } + .withMessageContaining(message) + + assertThat(contents(output)).isEqualTo(previous) + } + + @TableTest( + """ + scenario | tags | message + alias matches canonical | '[{dd-name: foo, otel-name: bar}, {dd-name: bar}]' | collides with canonical + alias matches own name | '[{dd-name: foo, otel-name: foo}]' | collides with canonical + alias shared by tags | '[{dd-name: foo, otel-name: alias}, {dd-name: bar, otel-name: alias}]' | claimed by both + duplicate declaration | '[{dd-name: foo, otel-name: first}, {dd-name: foo, otel-name: second}]' | declared in both + """ + ) + fun `duplicate declarations and ambiguous OpenTelemetry names are rejected`(tags: String, message: String) { + val yaml = directory.conventionsFile("span_types: {base: {abstract: true, tags: $tags}}") + + assertThatIllegalArgumentException() + .isThrownBy { TagRegistryGenerator.generate(yaml, File(directory, "generated")) } + .withMessageContaining(message) + } + + @TableTest( + """ + scenario | domain | message + self inheritance | span_types: {base: {extends: base}} | cyclic extends + cyclic inheritance | span_types: {a: {extends: b}, b: {extends: a}} | cyclic extends + unknown parent | span_types: {base: {extends: missing}} | extends unknown span type 'missing' + unknown mixin | span_types: {base: {include: [missing]}} | includes unknown mixin 'missing' + null span type | span_types: {base: null} | span type 'base' must be a mapping + null mixin | mixins: {peer: null} | mixin 'peer' must be a mapping + scalar applies | mixins: {peer: {applies: test}} | applies must be 'all' or a list + numeric applies | mixins: {peer: {applies: 42}} | applies must be 'all' or a list + nonstring applies | mixins: {peer: {applies: [42]}} | applies must be 'all' or a list + nonmapping span types | span_types: [] | span_types must be a mapping + nonmapping mixins | mixins: [] | mixins must be a mapping + nonmapping trace level | trace_level: [] | trace_level must be a mapping + scalar include | span_types: {base: {include: peer}} | include must be a list of mixin names + nonstring include | span_types: {base: {include: [42]}} | include must be a list of mixin names + list parent | span_types: {base: {extends: [base]}} | extends must be a span type name + numeric parent | span_types: {base: {extends: 42}} | extends must be a span type name + string abstract | span_types: {base: {abstract: 'true'}} | abstract must be a boolean + mapping tags | span_types: {base: {tags: {dd-name: x}}} | tags must be a list of tag declarations + scalar tag | span_types: {base: {tags: [foo]}} | tag declaration must be a mapping + empty reference | span_types: {base: {tags: [{ref: ''}]}} | ref has no valid tag name + nonstring reference required | span_types: {base: {tags: [{ref: foo, required: 42}]}} | required must be a string + scalar trace tag | trace_level: {tags: [foo]} | tag declaration must be a mapping + trace reference | trace_level: {tags: [{ref: foo}]} | trace_level tags must be declarations + """ + ) + fun `invalid composition preserves previous output`(domain: String, message: String) { + val yaml = directory.conventionsFile("span_types: {base: {tags: [{dd-name: valid}]}}") + val output = File(directory, "generated") + TagRegistryGenerator.generate(yaml, output) + val previous = contents(output) + directory.conventionsFile(domain) + + assertThatIllegalArgumentException() + .isThrownBy { TagRegistryGenerator.generate(yaml, output) } + .withMessageContaining(message) + + assertThat(contents(output)).isEqualTo(previous) + } + + @ParameterizedTest + @ValueSource(strings = ["url.full", "url.path"]) + fun `duplicate declarations across groups are rejected even when they agree`(otelName: String) { + val yaml = directory.conventionsFile( + """ + span_types: + client: + tags: [{dd-name: http.url, otel-name: url.full}] + server: + tags: [{dd-name: http.url, otel-name: $otelName}] + """ + ) + + assertThatIllegalArgumentException() + .isThrownBy { TagRegistryGenerator.generate(yaml, File(directory, "generated")) } + .withMessageContaining("declared in both 'client' and 'server'") + } + + @Test + fun `ref overrides only the requirement level on the referencing type`() { + val conv = tagConventions( + """ + span_types: + base: + abstract: true + tags: [{dd-name: http.url, type: string, required: required, otel-name: url.full}] + server: + extends: base + client: + extends: base + tags: [{ref: http.url, required: conditional}] + """ + ) + + val onClient = conv.resolve("client").single { it.name == "http.url" } + val onServer = conv.resolve("server").single { it.name == "http.url" } + assertThat(onClient.required).isEqualTo("conditional") + assertThat(onClient.otelName).isEqualTo("url.full") + assertThat(onServer.required).isEqualTo("required") + assertThat(conv.allDeclaredTags().map { it.name }).containsExactly("http.url") + } + + @Test + fun `ref adds a tag declared by an unrelated span type`() { + val conv = tagConventions( + """ + span_types: + server: + tags: [{dd-name: http.url, type: string, required: required}] + client: + tags: [{ref: http.url}] + """ + ) + + assertThat(conv.resolve("client").map { it.name to it.required }) + .containsExactly("http.url" to "required") + } + + @TableTest( + """ + scenario | tags | message + reference overrides alias | '[{dd-name: http.url, type: string}, {ref: http.url, otel-name: url.full}]' | may override only `required` + undeclared reference | '[{ref: http.url}]' | refs undeclared tag 'http.url' + """ + ) + fun `invalid refs are rejected`(tags: String, message: String) { + val yaml = directory.conventionsFile("span_types: {client: {tags: $tags}}") + + assertThatIllegalArgumentException() + .isThrownBy { TagRegistryGenerator.generate(yaml, File(directory, "generated")) } + .withMessageContaining(message) + } + + @Test + fun `rename on a concrete span type requires span-kind-neutral`() { + val yaml = directory.conventionsFile( + """ + span_types: + http.server: + tags: [{dd-name: http.hostname, otel-name: server.address}] + """ + ) + + assertThatIllegalArgumentException() + .isThrownBy { TagRegistryGenerator.generate(yaml, File(directory, "generated")) } + .withMessageContaining("on concrete span type 'http.server'") + .withMessageContaining("span-kind-neutral") + } + + @Test + fun `rename on a concrete span type passes when marked span-kind-neutral`() { + val yaml = directory.conventionsFile( + """ + span_types: + db.client: + tags: [{dd-name: db.type, otel-name: db.system, span-kind-neutral: true}] + """ + ) + val output = File(directory, "generated") + + TagRegistryGenerator.generate(yaml, output) + + assertThat(contents(output).getValue("java/datadog/trace/api/KnownTags.java")) + .contains("public static final String DB_TYPE_OTEL_NAME = \"db.system\";") + } + + @Test + fun `rename in a shared scope needs no span-kind-neutral`() { + val yaml = directory.conventionsFile( + """ + span_types: + http: + abstract: true + tags: [{dd-name: http.method, otel-name: http.request.method}] + http.server: + extends: http + mixins: + peer: + tags: [{dd-name: peer.port, type: int, otel-name: server.port}] + """ + ) + val output = File(directory, "generated") + + TagRegistryGenerator.generate(yaml, output) + + assertThat(contents(output).getValue("java/datadog/trace/api/KnownTags.java")) + .contains("HTTP_METHOD_OTEL_NAME", "PEER_PORT_OTEL_NAME") + } + + @Test + fun `span-kind-neutral without an otel-name fails`() { + val yaml = directory.conventionsFile( + """ + span_types: + web: + tags: [{dd-name: http.route, span-kind-neutral: true}] + """ + ) + + assertThatIllegalArgumentException() + .isThrownBy { TagRegistryGenerator.generate(yaml, File(directory, "generated")) } + .withMessageContaining("span-kind-neutral without an otel-name") + } + + private fun tagConventions(@Language("yaml") yamlText: String) = TagConventions.parse( + ObjectMapper(YAMLFactory()).readValue( + directory.conventionsFile(yamlText), + object : TypeReference>() {} + ) + ) + + private fun contents(output: File): Map = output.walkTopDown().filter { it.isFile } + .associate { + it.relativeTo(output).invariantSeparatorsPath to it.readText() + } +} diff --git a/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TestFiles.kt b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TestFiles.kt new file mode 100644 index 00000000000..bbf8560049e --- /dev/null +++ b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TestFiles.kt @@ -0,0 +1,11 @@ +package datadog.buildlogic.tagRegistry + +import org.intellij.lang.annotations.Language +import java.io.File + +internal fun File.writeFile(path: String, vararg contents: String) = resolve(path).apply { + parentFile.mkdirs() + writeText(contents.joinToString("\n") { it.trimIndent() } + "\n") +} + +internal fun File.conventionsFile(@Language("yaml") yamlText: String) = writeFile("tag-conventions.yaml", yamlText) diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 135ae0be3b9..08c8fd66c0e 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -104,6 +104,7 @@ dependencies { implementation(libs.asm) implementation(libs.asm.tree) + // Keep aligned with build-logic/tag-registry, which shares these Jackson classes. implementation(platform("com.fasterxml.jackson:jackson-bom:2.17.2")) implementation("com.fasterxml.jackson.core:jackson-databind") implementation("com.fasterxml.jackson.core:jackson-annotations") diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/java/opentelemetry14/OpenTelemetry14ConventionsTest.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/java/opentelemetry14/OpenTelemetry14ConventionsTest.java index 636df381731..51d4d59452a 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/java/opentelemetry14/OpenTelemetry14ConventionsTest.java +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/java/opentelemetry14/OpenTelemetry14ConventionsTest.java @@ -40,6 +40,16 @@ public class OpenTelemetry14ConventionsTest extends AbstractOpenTelemetry14Test private static final String SPAN_KIND_INTERNAL = "internal"; private static final String OPERATION_NAME_SPECIFIC_ATTRIBUTE = "operation.name"; + // Attributes with a dd-name registered under a different otel-name in tag-conventions.yaml are + // stored (and must be asserted) under their canonical Datadog name, not the raw OTel attribute + // key. + private static final Map CANONICAL_TAG_NAMES = new HashMap<>(); + + static { + CANONICAL_TAG_NAMES.put("http.request.method", "http.method"); + CANONICAL_TAG_NAMES.put("db.system", "db.type"); + } + static Stream testSpanNameConventionsArguments() { return Stream.of( // Fallback behavior @@ -121,7 +131,8 @@ void testSpanNameConventions( attributes.forEach( (key, value) -> { if (!OPERATION_NAME_SPECIFIC_ATTRIBUTE.equals(key)) { - tagMatchers.add(tag(key, is(value))); + String tagName = CANONICAL_TAG_NAMES.getOrDefault(key, key); + tagMatchers.add(tag(tagName, is(value))); } }); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index 1de59662c8d..09698ee0c5c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -10,6 +10,7 @@ import datadog.trace.api.DDTags; import datadog.trace.api.DDTraceId; import datadog.trace.api.Functions; +import datadog.trace.api.KnownTags; import datadog.trace.api.ProcessTags; import datadog.trace.api.TagMap; import datadog.trace.api.cache.DDCache; @@ -1237,6 +1238,7 @@ Object getTag(final String key) { // maintain previously observable type of the thread name :| return threadName.toString(); case Tags.HTTP_STATUS: + case KnownTags.HTTP_STATUS_CODE_OTEL_NAME: return 0 == httpStatusCode ? null : (int) httpStatusCode; case Tags.SPAN_KIND: return getSpanKindString(); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java index 845f870b965..17963da3514 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java @@ -9,7 +9,6 @@ import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.DOUBLE_ATTRIBUTE; import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.LONG_ATTRIBUTE; import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.STRING_ATTRIBUTE; -import static datadog.trace.common.writer.RemoteMapper.HTTP_STATUS; import static datadog.trace.common.writer.ddagent.TraceMapper.ORIGIN_KEY; import static datadog.trace.common.writer.ddagent.TraceMapper.PROCESS_TAGS_KEY; import static datadog.trace.common.writer.ddagent.TraceMapper.SAMPLING_PRIORITY_KEY; @@ -26,6 +25,8 @@ import datadog.json.JsonWriter; import datadog.trace.api.Config; import datadog.trace.api.DDTags; +import datadog.trace.api.KnownTagCodec; +import datadog.trace.api.KnownTags; import datadog.trace.api.TagMap; import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; @@ -41,11 +42,27 @@ /** Provides writers for OpenTelemetry's "trace.proto" JSON encoding. */ public final class OtlpTraceJson { - private static final UTF8BytesString SERVICE_NAME = UTF8BytesString.create("service.name"); private static final UTF8BytesString RESOURCE_NAME = UTF8BytesString.create("resource.name"); private static final UTF8BytesString OPERATION_NAME = UTF8BytesString.create("operation.name"); private static final UTF8BytesString SPAN_TYPE = UTF8BytesString.create("span.type"); + /* + * Span fields and intercepted Metadata fields bypass TagMap entry projection. Resolve their + * keys through the registry here so JSON and protobuf use the same OpenTelemetry names. + */ + private static final UTF8BytesString SERVICE_NAME_KEY = otelKey(KnownTags.SERVICE_ID); + + private static final UTF8BytesString HTTP_STATUS_CODE_KEY = + otelKey(KnownTags.HTTP_STATUS_CODE_ID); + + private static final UTF8BytesString HTTP_STATUS_CODE_KEY_DD = + UTF8BytesString.create(KnownTags.HTTP_STATUS_CODE_NAME); + + /** The OpenTelemetry-namespace key for a known tag, as named by the registry. */ + private static UTF8BytesString otelKey(long tagId) { + return UTF8BytesString.create(KnownTagCodec.openTelemetryTagOf(tagId)); + } + private OtlpTraceJson() {} /** Writes one complete {@code Span} JSON object. */ @@ -88,7 +105,7 @@ public static void writeSpan( writer.name("attributes").beginArray(); if (!Config.get().getServiceName().equals(span.getServiceName())) { - writeSpanTag(writer, SERVICE_NAME, span.getServiceName()); + writeSpanTag(writer, SERVICE_NAME_KEY, span.getServiceName()); } writeSpanTag(writer, RESOURCE_NAME, span.getResourceName()); writeSpanTag(writer, OPERATION_NAME, span.getOperationName()); @@ -143,20 +160,25 @@ public static void writeSpanLink(JsonWriter writer, AgentSpanLink spanLink) { } private static void writeSpanTag(JsonWriter writer, TagMap.EntryReader tagEntry) { + // OTLP is the OpenTelemetry wire format, but the rename itself is opt-in: emit the + // OpenTelemetry-namespace name only when OTel semantics are enabled, else the entry's own + // (Datadog) name, so existing consumers keep seeing Datadog names until they opt in. + String key = + Config.get().isTraceOtelSemanticsEnabled() ? tagEntry.openTelemetryTag() : tagEntry.tag(); switch (tagEntry.type()) { case TagMap.EntryReader.BOOLEAN: - writeAttribute(writer, BOOLEAN_ATTRIBUTE, tagEntry.tag(), tagEntry.objectValue()); + writeAttribute(writer, BOOLEAN_ATTRIBUTE, key, tagEntry.objectValue()); break; case TagMap.EntryReader.INT: case TagMap.EntryReader.LONG: - writeAttribute(writer, LONG_ATTRIBUTE, tagEntry.tag(), tagEntry.objectValue()); + writeAttribute(writer, LONG_ATTRIBUTE, key, tagEntry.objectValue()); break; case TagMap.EntryReader.FLOAT: case TagMap.EntryReader.DOUBLE: - writeAttribute(writer, DOUBLE_ATTRIBUTE, tagEntry.tag(), tagEntry.objectValue()); + writeAttribute(writer, DOUBLE_ATTRIBUTE, key, tagEntry.objectValue()); break; default: - writeAttribute(writer, STRING_ATTRIBUTE, tagEntry.tag(), tagEntry.stringValue()); + writeAttribute(writer, STRING_ATTRIBUTE, key, tagEntry.stringValue()); } } @@ -211,7 +233,12 @@ public void accept(Metadata metadata) { writeSpanTag(writer, THREAD_ID, metadata.getThreadId()); writeSpanTag(writer, THREAD_NAME, metadata.getThreadName()); if (metadata.getHttpStatusCode() != UNSET_STATUS) { - writeSpanTag(writer, HTTP_STATUS, metadata.getHttpStatusCodeString()); + writeSpanTag( + writer, + Config.get().isTraceOtelSemanticsEnabled() + ? HTTP_STATUS_CODE_KEY + : HTTP_STATUS_CODE_KEY_DD, + metadata.getHttpStatusCode()); } if (metadata.getOrigin() != null) { writeSpanTag(writer, ORIGIN_KEY, metadata.getOrigin()); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java index 707225cf5a6..e7b0310bc42 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java @@ -9,7 +9,6 @@ import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.DOUBLE_ATTRIBUTE; import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.LONG_ATTRIBUTE; import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.STRING_ATTRIBUTE; -import static datadog.trace.common.writer.RemoteMapper.HTTP_STATUS; import static datadog.trace.common.writer.ddagent.TraceMapper.ORIGIN_KEY; import static datadog.trace.common.writer.ddagent.TraceMapper.PROCESS_TAGS_KEY; import static datadog.trace.common.writer.ddagent.TraceMapper.SAMPLING_PRIORITY_KEY; @@ -38,6 +37,8 @@ import datadog.trace.api.Config; import datadog.trace.api.DDTags; import datadog.trace.api.DDTraceId; +import datadog.trace.api.KnownTagCodec; +import datadog.trace.api.KnownTags; import datadog.trace.api.TagMap; import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; @@ -53,11 +54,27 @@ /** Provides optimized writers for OpenTelemetry's "trace.proto" wire protocol. */ public final class OtlpTraceProto { - private static final UTF8BytesString SERVICE_NAME = UTF8BytesString.create("service.name"); private static final UTF8BytesString RESOURCE_NAME = UTF8BytesString.create("resource.name"); private static final UTF8BytesString OPERATION_NAME = UTF8BytesString.create("operation.name"); private static final UTF8BytesString SPAN_TYPE = UTF8BytesString.create("span.type"); + /* + * Span fields and intercepted Metadata fields bypass TagMap entry projection. Resolve their + * keys through the registry here so OpenTelemetry names stay consistent with entry tags. + */ + private static final UTF8BytesString SERVICE_NAME_KEY = otelKey(KnownTags.SERVICE_ID); + + private static final UTF8BytesString HTTP_STATUS_CODE_KEY = + otelKey(KnownTags.HTTP_STATUS_CODE_ID); + + private static final UTF8BytesString HTTP_STATUS_CODE_KEY_DD = + UTF8BytesString.create(KnownTags.HTTP_STATUS_CODE_NAME); + + /** The OpenTelemetry-namespace key for a known tag, as named by the registry. */ + private static UTF8BytesString otelKey(long tagId) { + return UTF8BytesString.create(KnownTagCodec.openTelemetryTagOf(tagId)); + } + private OtlpTraceProto() {} /** Records a scoped spans message after its nested span messages have been recorded. */ @@ -134,7 +151,7 @@ public static int recordSpanMessage( writeI64(buf, span.getStartTime() + PendingTrace.getDurationNano(span)); if (!Config.get().getServiceName().equals(span.getServiceName())) { - writeSpanTag(buf, SERVICE_NAME, span.getServiceName()); + writeSpanTag(buf, SERVICE_NAME_KEY, span.getServiceName()); } writeSpanTag(buf, RESOURCE_NAME, span.getResourceName()); writeSpanTag(buf, OPERATION_NAME, span.getOperationName()); @@ -208,20 +225,25 @@ public static void writeSpanId(StreamingBuffer buf, long spanId) { private static void writeSpanTag(StreamingBuffer buf, TagMap.EntryReader tagEntry) { writeTag(buf, 9, LEN_WIRE_TYPE); + // OTLP is the OpenTelemetry wire format, but the rename itself is opt-in: emit the + // OpenTelemetry-namespace name only when OTel semantics are enabled, else the entry's own + // (Datadog) name, so existing consumers keep seeing Datadog names until they opt in. + String key = + Config.get().isTraceOtelSemanticsEnabled() ? tagEntry.openTelemetryTag() : tagEntry.tag(); switch (tagEntry.type()) { case TagMap.EntryReader.BOOLEAN: - writeAttribute(buf, BOOLEAN_ATTRIBUTE, tagEntry.tag(), tagEntry.objectValue()); + writeAttribute(buf, BOOLEAN_ATTRIBUTE, key, tagEntry.objectValue()); break; case TagMap.EntryReader.INT: case TagMap.EntryReader.LONG: - writeAttribute(buf, LONG_ATTRIBUTE, tagEntry.tag(), tagEntry.objectValue()); + writeAttribute(buf, LONG_ATTRIBUTE, key, tagEntry.objectValue()); break; case TagMap.EntryReader.FLOAT: case TagMap.EntryReader.DOUBLE: - writeAttribute(buf, DOUBLE_ATTRIBUTE, tagEntry.tag(), tagEntry.objectValue()); + writeAttribute(buf, DOUBLE_ATTRIBUTE, key, tagEntry.objectValue()); break; default: - writeAttribute(buf, STRING_ATTRIBUTE, tagEntry.tag(), tagEntry.stringValue()); + writeAttribute(buf, STRING_ATTRIBUTE, key, tagEntry.stringValue()); } } @@ -288,7 +310,12 @@ public void accept(Metadata metadata) { writeSpanTag(buf, THREAD_ID, metadata.getThreadId()); writeSpanTag(buf, THREAD_NAME, metadata.getThreadName()); if (metadata.getHttpStatusCode() != UNSET_STATUS) { - writeSpanTag(buf, HTTP_STATUS, metadata.getHttpStatusCodeString()); + writeSpanTag( + buf, + Config.get().isTraceOtelSemanticsEnabled() + ? HTTP_STATUS_CODE_KEY + : HTTP_STATUS_CODE_KEY_DD, + metadata.getHttpStatusCode()); } if (metadata.getOrigin() != null) { writeSpanTag(buf, ORIGIN_KEY, metadata.getOrigin()); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/taginterceptor/TagInterceptor.java b/dd-trace-core/src/main/java/datadog/trace/core/taginterceptor/TagInterceptor.java index d81a9cc8441..89e5a01adb8 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/taginterceptor/TagInterceptor.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/taginterceptor/TagInterceptor.java @@ -4,6 +4,10 @@ import static datadog.trace.api.DDTags.MEASURED; import static datadog.trace.api.DDTags.ORIGIN_KEY; import static datadog.trace.api.DDTags.SPAN_TYPE; +import static datadog.trace.api.KnownTags.DB_STATEMENT_OTEL_NAME; +import static datadog.trace.api.KnownTags.HTTP_METHOD_OTEL_NAME; +import static datadog.trace.api.KnownTags.HTTP_STATUS_CODE_OTEL_NAME; +import static datadog.trace.api.KnownTags.HTTP_URL_OTEL_NAME; import static datadog.trace.api.sampling.PrioritySampling.USER_DROP; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.SERVLET_CONTEXT; import static datadog.trace.bootstrap.instrumentation.api.ServiceNameSources.SPLIT_BY_SERVLET_CONTEXT; @@ -103,6 +107,7 @@ public boolean needsIntercept(String tag) { switch (tag) { case DDTags.RESOURCE_NAME: case Tags.DB_STATEMENT: + case DB_STATEMENT_OTEL_NAME: case DDTags.SERVICE_NAME: case "service": case Tags.PEER_SERVICE: @@ -118,8 +123,11 @@ public boolean needsIntercept(String tag) { case ANALYTICS_SAMPLE_RATE: case Tags.ERROR: case HTTP_STATUS: + case HTTP_STATUS_CODE_OTEL_NAME: case HTTP_METHOD: + case HTTP_METHOD_OTEL_NAME: case HTTP_URL: + case HTTP_URL_OTEL_NAME: case ORIGIN_KEY: case MEASURED: case Tags.SPAN_KIND: @@ -135,6 +143,7 @@ public boolean interceptTag(DDSpanContext span, String tag, Object value) { case DDTags.RESOURCE_NAME: return interceptResourceName(span, value); case Tags.DB_STATEMENT: + case DB_STATEMENT_OTEL_NAME: return interceptDbStatement(span, value); case DDTags.SERVICE_NAME: case "service": @@ -184,10 +193,13 @@ public boolean interceptTag(DDSpanContext span, String tag, Object value) { case Tags.ERROR: return interceptError(span, value); case HTTP_STATUS: + case HTTP_STATUS_CODE_OTEL_NAME: // not set internally but may come from manual instrumentation return interceptHttpStatusCode(span, value); case HTTP_METHOD: + case HTTP_METHOD_OTEL_NAME: case HTTP_URL: + case HTTP_URL_OTEL_NAME: return interceptUrlResourceAsNameRule(span, tag, value); case ORIGIN_KEY: return interceptOrigin(span, value); @@ -205,13 +217,17 @@ public boolean interceptTag(DDSpanContext span, String tag, Object value) { private boolean interceptUrlResourceAsNameRule(DDSpanContext span, String tag, Object value) { if (shouldSetUrlResourceAsName) { - if (HTTP_METHOD.equals(tag)) { + if (HTTP_METHOD.equals(tag) || HTTP_METHOD_OTEL_NAME.equals(tag)) { final Object url = span.unsafeGetTag(HTTP_URL); if (url != null) { setResourceFromUrl(span, value.toString(), url); } - } else if (HTTP_URL.equals(tag)) { - final Object method = span.unsafeGetTag(HTTP_METHOD); + } else if (HTTP_URL.equals(tag) || HTTP_URL_OTEL_NAME.equals(tag)) { + // the method may have been set under either spelling -- see HTTP_METHOD_OTEL_NAME. + Object method = span.unsafeGetTag(HTTP_METHOD); + if (method == null) { + method = span.unsafeGetTag(HTTP_METHOD_OTEL_NAME); + } setResourceFromUrl(span, method != null ? method.toString() : null, value); } } diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollectorTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollectorTest.java index 3f836227a39..a0456e9d20e 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollectorTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollectorTest.java @@ -13,9 +13,11 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import datadog.json.JsonMapper; +import datadog.trace.api.Config; import datadog.trace.api.DDTraceId; import datadog.trace.api.TracePropagationStyle; import datadog.trace.api.sampling.PrioritySampling; @@ -38,6 +40,8 @@ import java.util.Map; import java.util.Set; import org.junit.jupiter.api.Test; +import org.mockito.AdditionalAnswers; +import org.mockito.MockedStatic; /** * Tests for {@link OtlpTraceJsonCollector}, parsing the produced JSON back with {@link JsonMapper} @@ -77,6 +81,63 @@ void singleSpanIsEncodedWithHexIdsAndCamelCaseKeys() throws IOException { assertTrue(attrKeys.contains("operation.name")); } + @Test + void tagsAreEmittedUnderTheirOpenTelemetryNameWhenOtelSemanticsEnabled() throws IOException { + // The JSON encoder is a second exporter of the same spans, so it must apply the registry's + // OpenTelemetry naming exactly as the protobuf one does: which transport protocol is configured + // must not change the attribute names a backend receives. + Set attrKeys = collectTagsAttributeKeys(true); + + assertTrue( + attrKeys.contains("http.request.method"), + "renamed tag must use its OpenTelemetry name; got " + attrKeys); + assertFalse( + attrKeys.contains("http.method"), + "renamed tag must not also appear under its Datadog name; got " + attrKeys); + assertTrue( + attrKeys.contains("custom.unregistered"), + "a tag the registry does not name passes through unchanged; got " + attrKeys); + } + + @Test + void tagsAreEmittedUnderTheirDatadogNameWhenOtelSemanticsDisabled() throws IOException { + // The OpenTelemetry rename is opt-in: with the flag off, existing consumers must keep seeing + // Datadog names, not the OpenTelemetry ones. + Set attrKeys = collectTagsAttributeKeys(false); + + assertTrue( + attrKeys.contains("http.method"), + "tag must use its Datadog name when OTel semantics are disabled; got " + attrKeys); + assertFalse( + attrKeys.contains("http.request.method"), + "tag must not appear under its OpenTelemetry name when OTel semantics are disabled; got " + + attrKeys); + assertTrue( + attrKeys.contains("custom.unregistered"), + "a tag the registry does not name passes through unchanged; got " + attrKeys); + } + + private static Set collectTagsAttributeKeys(boolean otelSemanticsEnabled) + throws IOException { + Config realConfig = Config.get(); + try (MockedStatic configMock = mockStatic(Config.class)) { + Config config = mock(Config.class, AdditionalAnswers.delegatesTo(realConfig)); + when(config.isTraceOtelSemanticsEnabled()).thenReturn(otelSemanticsEnabled); + configMock.when(Config::get).thenReturn(config); + + AgentSpan agentSpan = TRACER.startSpan("test", "op.tagged"); + agentSpan.setResourceName("GET /api"); + agentSpan.setTag("http.method", "GET"); + agentSpan.setTag("custom.unregistered", "value"); + agentSpan.setSamplingPriority(PrioritySampling.USER_KEEP, SamplingMechanism.DEFAULT); + agentSpan.finish(); + + OtlpTraceJsonCollector collector = new OtlpTraceJsonCollector(); + collector.addTrace(asList((CoreSpan) agentSpan)); + return attributeKeys(onlySpan(collector.collectTraces())); + } + } + @Test void spanKindIsEncodedAsInteger() throws IOException { DDSpan span = startAndFinish("op.server", "GET /api", SPAN_KIND_SERVER); diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceProtoTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceProtoTest.java index e326a6958b8..1402c16c7b3 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceProtoTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceProtoTest.java @@ -20,12 +20,15 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import com.google.protobuf.CodedInputStream; import com.google.protobuf.WireFormat; +import datadog.trace.api.Config; import datadog.trace.api.DD128bTraceId; import datadog.trace.api.DDTraceId; +import datadog.trace.api.KnownTagCodec; import datadog.trace.api.TracePropagationStyle; import datadog.trace.api.sampling.PrioritySampling; import datadog.trace.api.sampling.SamplingMechanism; @@ -51,6 +54,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.AdditionalAnswers; +import org.mockito.MockedStatic; /** * Tests for {@link OtlpTraceProto} via {@link OtlpTraceProtoCollector#collectTraces}. @@ -457,11 +462,43 @@ static Stream cases() { kindSpan("third.span", SPAN_KIND_SERVER)))); } + /** + * Cartesian product of {@link #cases()} with both settings of the OTel-semantics flag, so every + * case is verified both with the OpenTelemetry-namespace renames applied and with them off (the + * default): the rename is opt-in, so both states must serialize correctly. + */ + @SuppressWarnings("unchecked") + static Stream casesWithOtelSemantics() { + return cases() + .flatMap( + args -> { + Object[] raw = args.get(); + String caseName = (String) raw[0]; + List specs = (List) raw[1]; + return Stream.of( + Arguments.of(caseName + " [otel semantics enabled]", specs, true), + Arguments.of(caseName + " [otel semantics disabled]", specs, false)); + }); + } + // ── parameterized test ──────────────────────────────────────────────────── @ParameterizedTest(name = "{0}") - @MethodSource("cases") - void testCollectTraces(String caseName, List specs) throws IOException { + @MethodSource("casesWithOtelSemantics") + void testCollectTraces(String caseName, List specs, boolean otelSemanticsEnabled) + throws IOException { + Config realConfig = Config.get(); + try (MockedStatic configMock = mockStatic(Config.class)) { + Config config = mock(Config.class, AdditionalAnswers.delegatesTo(realConfig)); + when(config.isTraceOtelSemanticsEnabled()).thenReturn(otelSemanticsEnabled); + configMock.when(Config::get).thenReturn(config); + + testCollectTracesImpl(caseName, specs, otelSemanticsEnabled); + } + } + + private void testCollectTracesImpl( + String caseName, List specs, boolean otelSemanticsEnabled) throws IOException { List spans = buildSpans(specs); OtlpTraceProtoCollector collector = new OtlpTraceProtoCollector(); @@ -527,7 +564,11 @@ void testCollectTraces(String caseName, List specs) throws IOException // ── verify each span ───────────────────────────────────────────────── for (int i = 0; i < spans.size(); i++) { verifySpan( - CodedInputStream.newInstance(spanBlobs.get(i)), spans.get(i), specs.get(i), caseName); + CodedInputStream.newInstance(spanBlobs.get(i)), + spans.get(i), + specs.get(i), + caseName, + otelSemanticsEnabled); } } @@ -999,7 +1040,11 @@ private static void verifyDefaultScope(CodedInputStream scope) throws IOExceptio * */ private static void verifySpan( - CodedInputStream spanData, DDSpan originalSpan, SpanSpec spec, String caseName) + CodedInputStream spanData, + DDSpan originalSpan, + SpanSpec spec, + String caseName, + boolean otelSemanticsEnabled) throws IOException { byte[] parsedTraceId = null; byte[] parsedSpanId = null; @@ -1167,11 +1212,67 @@ private static void verifySpan( "attributes must include 'service.name' when service is overridden [" + caseName + "]"); } - // extra user tags must appear as attributes + // Assert the selected namespace spelling and reject the other. HTTP status is checked + // separately because interception moves it out of the TagMap before serialization. for (String key : spec.extraTags.keySet()) { + if ("http.status_code".equals(key)) { + // Not a tag-map entry by the time it is serialized: the set path intercepts it into + // Metadata.httpStatusCode, so it never reaches the per-entry projection and instead is + // resolved through the fixed HTTP_STATUS_CODE_KEY constant (see the intercepted-status + // assertion below). + String expectedStatusKey = + otelSemanticsEnabled ? "http.response.status_code" : "http.status_code"; + String otherStatusKey = + otelSemanticsEnabled ? "http.status_code" : "http.response.status_code"; + assertTrue( + attrKeys.contains(expectedStatusKey), + "intercepted status must be emitted as '" + + expectedStatusKey + + "' [" + + caseName + + "]; got " + + attrKeys); + assertFalse( + attrKeys.contains(otherStatusKey), + "intercepted status must not also appear as '" + + otherStatusKey + + "' [" + + caseName + + "]"); + continue; + } + long id = KnownTagCodec.keyOf(key); + String otelName = id != 0L ? KnownTagCodec.openTelemetryNameOf(id) : null; + String expected = otelSemanticsEnabled && otelName != null ? otelName : key; assertTrue( - attrKeys.contains(key), - "attributes must include extra tag '" + key + "' [" + caseName + "]"); + attrKeys.contains(expected), + "attributes must include extra tag '" + + key + + "' as '" + + expected + + "' [" + + caseName + + "]; got " + + attrKeys); + if (otelSemanticsEnabled && otelName != null) { + assertFalse( + attrKeys.contains(key), + "renamed tag '" + + key + + "' must not also appear under its Datadog name [" + + caseName + + "]"); + } else if (otelName != null) { + assertFalse( + attrKeys.contains(otelName), + "tag '" + + key + + "' must not appear under its OpenTelemetry name '" + + otelName + + "' when OTel semantics are disabled [" + + caseName + + "]"); + } } if (spec.measured) { @@ -1180,11 +1281,26 @@ private static void verifySpan( "attributes must include '_dd.measured' for measured spans [" + caseName + "]"); } if (spec.httpStatusCode != 0) { + // Intercepted into Metadata.httpStatusCode rather than left in the tag map, so its name comes + // from a key constant in OtlpTraceProto (HTTP_STATUS_CODE_KEY / HTTP_STATUS_CODE_KEY_DD) and + // not from the per-entry projection. Emitted as an int attribute, matching the + // semantic-conventions type, since Metadata now carries the status as an int; under its + // OpenTelemetry name only when OTel semantics are enabled, else its Datadog name. + String expectedStatusKey = + otelSemanticsEnabled ? "http.response.status_code" : "http.status_code"; + String otherStatusKey = + otelSemanticsEnabled ? "http.status_code" : "http.response.status_code"; assertTrue( - attrKeys.contains("http.status_code"), - "attributes must include 'http.status_code' when set via setHttpStatusCode [" + attrKeys.contains(expectedStatusKey), + "attributes must include '" + + expectedStatusKey + + "' when set via setHttpStatusCode [" + caseName - + "]"); + + "]; got " + + attrKeys); + assertFalse( + attrKeys.contains(otherStatusKey), + "status code must not also be emitted as '" + otherStatusKey + "' [" + caseName + "]"); } if (spec.origin != null) { assertTrue( diff --git a/dd-trace-core/src/test/java/datadog/trace/core/taginterceptor/TagInterceptorTest.java b/dd-trace-core/src/test/java/datadog/trace/core/taginterceptor/TagInterceptorTest.java index ae85244a729..91f3da0910f 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/taginterceptor/TagInterceptorTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/taginterceptor/TagInterceptorTest.java @@ -707,6 +707,90 @@ static Stream urlAsResourceNameRuleSetsTheResourceNameArguments() { ); } + @TableTest({ + "scenario | methodTag ", + "datadog spelling | 'Tags.HTTP_METHOD' ", + "otel spelling | 'http.request.method'" + }) + void urlAsResourceNameRuleAppliesRegardlessOfMethodSpellingWhenUrlSetFirst( + @ConvertWith(TagsConverter.class) String methodTag) { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + AgentSpan span = tracer.buildSpan("datadog", "fakeOperation").start(); + try { + span.setTag(HTTP_URL, "/with-method"); + span.setTag(methodTag, "Post"); + assertEquals("POST /with-method", span.getResourceName().toString()); + } finally { + span.finish(); + } + } + + @TableTest({ + "scenario | methodTag ", + "datadog spelling | 'Tags.HTTP_METHOD' ", + "otel spelling | 'http.request.method'" + }) + void urlAsResourceNameRuleAppliesRegardlessOfMethodSpellingWhenMethodSetFirst( + @ConvertWith(TagsConverter.class) String methodTag) { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + AgentSpan span = tracer.buildSpan("datadog", "fakeOperation").start(); + try { + span.setTag(methodTag, "Post"); + span.setTag(HTTP_URL, "/with-method"); + assertEquals("POST /with-method", span.getResourceName().toString()); + } finally { + span.finish(); + } + } + + @TableTest({ + "scenario | statementTag | beforeStart", + "datadog spelling, builder | 'db.statement' | true ", + "datadog spelling, setTag | 'db.statement' | false ", + "otel spelling, builder | 'db.query.text' | true ", + "otel spelling, setTag | 'db.query.text' | false " + }) + void dbStatementIsConsumedIntoResourceRegardlessOfSpellingOrTiming( + String statementTag, boolean beforeStart) { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + AgentSpan span = + beforeStart + ? tracer.buildSpan("datadog", "fakeOperation").withTag(statementTag, "select 1").start() + : tracer.buildSpan("datadog", "fakeOperation").start(); + try { + if (!beforeStart) { + span.setTag(statementTag, "select 1"); + } + assertEquals("select 1", span.getResourceName().toString()); + assertNull(span.getTag(Tags.DB_STATEMENT)); + assertNull(span.getTag("db.query.text")); + } finally { + span.finish(); + } + } + + @TableTest({ + "scenario | urlTag ", + "datadog spelling | 'Tags.HTTP_URL'", + "otel spelling | 'url.full' " + }) + void urlAsResourceNameRuleAppliesRegardlessOfUrlSpelling( + @ConvertWith(TagsConverter.class) String urlTag) { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + AgentSpan span = tracer.buildSpan("datadog", "fakeOperation").start(); + try { + span.setTag(HTTP_METHOD, "Post"); + span.setTag(urlTag, "/with-method"); + assertEquals("POST /with-method", span.getResourceName().toString()); + } finally { + span.finish(); + } + } + @Test void whenUserSetsPeerServiceTheSourceShouldBePeerService() { CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); @@ -720,6 +804,51 @@ void whenUserSetsPeerServiceTheSourceShouldBePeerService() { } } + @TableTest({ + "scenario | tag ", + "datadog spelling | 'Tags.HTTP_STATUS' ", + "otel spelling | 'http.response.status_code'" + }) + void httpStatusCodeIsInterceptedRegardlessOfSpelling( + @ConvertWith(TagsConverter.class) String tag) { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + DDSpan span = (DDSpan) tracer.buildSpan("datadog", "fakeOperation").start(); + try { + span.setTag(tag, 200); + assertEquals((short) 200, span.getHttpStatusCode()); + // consumed into Metadata's httpStatusCode field, not left behind as a generic tag under + // either spelling -- otherwise the OTLP writers would emit it a second time. getTag(...) + // is not a suitable check here: it synthesizes the Datadog spelling's value straight from + // httpStatusCode regardless of generic storage, so go straight to the backing TagMap. + assertNull(span.unsafeGetTag(HTTP_STATUS)); + assertNull(span.unsafeGetTag("http.response.status_code")); + } finally { + span.finish(); + } + } + + @TableTest({ + "scenario | tag ", + "datadog spelling | 'Tags.HTTP_STATUS' ", + "otel spelling | 'http.response.status_code'" + }) + void httpStatusCodeIsReadableRegardlessOfSpelling(@ConvertWith(TagsConverter.class) String tag) { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + DDSpan span = (DDSpan) tracer.buildSpan("datadog", "fakeOperation").start(); + try { + span.setTag(tag, 200); + // regardless of which spelling was used to set it, both spellings must read it back -- + // TagsMatcher (and thus trace-sampling rules) reads tags through getTag(), so a spelling + // the getter doesn't recognize would look unset even though it was intercepted. + assertEquals(200, span.getTag(Tags.HTTP_STATUS)); + assertEquals(200, span.getTag("http.response.status_code")); + } finally { + span.finish(); + } + } + @Test void whenInterceptServiceNameExtraServiceProviderIsCalled() { ServiceNameCollector origServiceNameCollector = ServiceNameCollector.get(); diff --git a/gradle/java_no_deps.gradle b/gradle/java_no_deps.gradle index d2154d39dad..b723dd5e73c 100644 --- a/gradle/java_no_deps.gradle +++ b/gradle/java_no_deps.gradle @@ -226,7 +226,7 @@ tasks.named("javadoc", Javadoc) { javadocTool = javaToolchains.javadocToolFor { languageVersion = JavaLanguageVersion.of(8) } - source = sourceSets.main.java.srcDirs + source = sourceSets.main.java classpath = configurations.compileClasspath options { diff --git a/internal-api/build.gradle.kts b/internal-api/build.gradle.kts index 7ae9ee327f4..a9d82cebc72 100644 --- a/internal-api/build.gradle.kts +++ b/internal-api/build.gradle.kts @@ -5,6 +5,7 @@ plugins { `java-library` id("dd-trace-java.module.internal-api") id("dd-trace-java.jmh-conventions") + id("dd-trace-java.tag-registry-generator") } java { @@ -26,6 +27,9 @@ extra["minimumBranchCoverage"] = 0.7 extra["minimumInstructionCoverage"] = 0.8 extra["excludedClassesCoverage"] = listOf( + // Generated by the tag-registry code generator. + "datadog.trace.api.KnownTags", + "datadog.trace.api.KnownTags.*", "datadog.trace.api.ClassloaderConfigurationOverrides", "datadog.trace.api.ClassloaderConfigurationOverrides.Lazy", // Interface @@ -260,6 +264,10 @@ extra["excludedClassesBranchCoverage"] = listOf( extra["excludedClassesInstructionCoverage"] = listOf("datadog.trace.util.stacktrace.StackWalkerFactory") +tagRegistry { + tagConventionsFile.set(rootProject.layout.projectDirectory.file("tag-conventions.yaml")) +} + dependencies { // references TraceScope and Continuation from public api api(project(":dd-trace-api")) diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java new file mode 100644 index 00000000000..d84e1e6a537 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -0,0 +1,140 @@ +package datadog.trace.api; + +/** + * Resolves generated tag IDs and names. {@link KnownTags} owns the lookup tables; this class owns + * the ID layout and namespace naming policy. + * + *

Datadog and OpenTelemetry names can resolve to the same ID. {@link #canonicalTagName(String)} + * provides the Datadog key used by {@link TagMap}; the namespace readers choose output names. The + * ID does not determine whether a tag is intercepted or where it is stored. + */ +public final class KnownTagCodec { + /* + * tagId bit layout: [63-48 serialNum (16 bits)] [47-32 reserved, zero] [31-0 flags]. serialNum is + * globally unique per known tag and is the whole of the tag's identity — nameOf/ + * openTelemetryNameOf switch on it, and the generator emits each id as a literal. Bits [47-32] + * are RESERVED and always zero here: they are the window the dense tag store uses for its + * co-occurrence slot coordinate, which arrives with that store. Of the low 32 flag bits, bit 2 is + * the trace/span LEVEL bit (set ⟹ trace-level); bits 1-0 are reserved. Unknown (string-only) + * custom tags are NOT known ids — {@code keyOf} returns 0 for them. + * + *

An id says what a tag IS, not how it is SET. Whether the tracer intercepts a tag on the + * set-path — routing it to a span field or a sampling directive instead of tag storage — belongs + * to TagInterceptor, whose {@code needsIntercept} switch is the authority; mirroring it here as a + * classification bit and a serial-range tier only created drift between the two. That + * classification returns with the work that consumes it (the id→handler dispatch table that + * retires TagInterceptor), and re-adding a bit then is purely additive. + * + *

There is deliberately NO OpenTelemetry-applicability flag: an absent otel-name means + * pass-through (the tag is emitted under its Datadog name), so today every known tag has an + * OpenTelemetry name and such a flag would be constant. It returns once a Datadog-only tag exists. + */ + public static int serialNum(long tagId) { + return (int) (tagId >>> 48); + } + + /** + * Trace/span LEVEL bit (low-32 carve, bit 2). Set marks a trace-level tag (lives on the + * TraceSegment's own TagMap); clear marks a span-level tag. Declared in the conventions as the + * {@code trace_level} tier, so it is part of the tag's identity rather than of any storage + * scheme. + */ + public static final long LEVEL_TRACE = 1L << 2; + + /** True if the tagId names a trace-level tag. */ + public static boolean isTraceLevel(long tagId) { + return (tagId & LEVEL_TRACE) != 0L; + } + + /** Returns the tagId with the {@link #LEVEL_TRACE} flag set. */ + public static long traceLevel(long tagId) { + return tagId | LEVEL_TRACE; + } + + /** + * Builds a tagId from its {@code serialNum} (globally unique per known tag). The reserved [47-32] + * window and the low 32 bits are zero, so the id is fully determined by the serial — the + * generator emits it as a literal. Inverse of {@link #serialNum}. Intended for the code generator + * and tests. + */ + public static long makeTagId(int serialNum) { + return (long) serialNum << 48; + } + + public interface Resolver { + /** The tag's Datadog-namespace (canonical) name. */ + String nameOf(long tagId); + + /** The tag's OpenTelemetry-namespace name, or {@code null} when it declares none. */ + String openTelemetryNameOf(long tagId); + + /** The id for {@code name} in ANY namespace (many→one), or 0 when it is not a known tag. */ + long keyOf(String name); + } + + /** + * Loads the generated resolver on the first name lookup. Reading {@code RESOLVER} initializes + * {@link KnownTags}, so callers cannot observe an unregistered or partially initialized registry. + * The {@code static final} receiver can also help the JIT inline resolver calls. + */ + private static final class Installed { + static final Resolver RESOLVER = KnownTags.RESOLVER; + } + + /** The tag's canonical (Datadog-namespace) name, or {@code null} when the id is not known. */ + public static String nameOf(long tagId) { + return Installed.RESOLVER.nameOf(tagId); + } + + /** The tag's Datadog-namespace (canonical) name — the same value as {@link #nameOf}. */ + public static String datadogNameOf(long tagId) { + return nameOf(tagId); + } + + /** + * The tag's declared OpenTelemetry RENAME, or {@code null} when it declares none. Raw registry + * data — it does not apply the pass-through default, so most callers want {@link + * #openTelemetryTagOf} instead. + */ + public static String openTelemetryNameOf(long tagId) { + return Installed.RESOLVER.openTelemetryNameOf(tagId); + } + + /** + * The name {@code tagId} is emitted under in the OpenTelemetry namespace: its declared rename + * when it has one, otherwise its Datadog name — pass-through, the default. {@code null} for an + * unknown id, which has no registry name at all; a custom tag falls back to its own key, and only + * the caller holding that key can do so. + * + *

This is the one place the pass-through policy lives, so no serializer re-decides it. Pair it + * with {@link #datadogNameOf} for the same tag under the Datadog namespace; outbound naming is + * per-namespace, never normalized to one of them. + */ + public static String openTelemetryTagOf(long tagId) { + Resolver resolver = Installed.RESOLVER; + String otelName = resolver.openTelemetryNameOf(tagId); + return otelName != null ? otelName : resolver.nameOf(tagId); + } + + /** The id for {@code name} in any namespace, or 0 when it is not a known tag. */ + public static long keyOf(String name) { + return Installed.RESOLVER.keyOf(name); + } + + /** + * The Datadog-namespace name to store {@code name} under: {@code name} itself when it is not a + * known tag, otherwise the canonical Datadog name for whichever id it resolves to. A Datadog name + * maps to itself (no-op); an OpenTelemetry rename maps to the Datadog name it is a rename of. + * + *

{@link TagMap} calls this at entry construction so that setting a known tag under its + * Datadog name and under its OpenTelemetry rename store to the same {@code Entry} rather than two + * separate ones -- the two names denote one tag, and only serialization ({@link #datadogNameOf}, + * {@link #openTelemetryTagOf}) should see them as different. + */ + public static String canonicalTagName(String name) { + long id = keyOf(name); + return id != 0 ? nameOf(id) : name; + } + + private KnownTagCodec() {} +} diff --git a/internal-api/src/main/java/datadog/trace/api/TagMap.java b/internal-api/src/main/java/datadog/trace/api/TagMap.java index 2f2423d5239..52d2ef32a2b 100644 --- a/internal-api/src/main/java/datadog/trace/api/TagMap.java +++ b/internal-api/src/main/java/datadog/trace/api/TagMap.java @@ -53,6 +53,10 @@ public final class TagMap implements Map, Iterable. public static final TagMap EMPTY = new TagMap(new Object[1], 0); + // Sentinel for an ID not yet resolved by the reused EntryReadingHelper. Zero denotes an unknown + // tag, so it cannot represent "not computed". + static final long TAG_ID_NOT_COMPUTED = Long.MIN_VALUE; + /** Creates a new mutable TagMap that contains the contents of map */ public static final TagMap fromMap(@Nonnull Map map) { TagMap tagMap = TagMap.create(map.size()); @@ -141,7 +145,9 @@ public final boolean matches(String tag) { public static final class EntryRemoval extends EntryChange { EntryRemoval(String tag) { - super(tag); + // Canonicalize so a removal recorded under an OpenTelemetry rename matches an Entry recorded + // (via Entry's own constructor) under its Datadog name -- see Ledger#contains/#matches. + super(KnownTagCodec.canonicalTagName(tag)); } @Override @@ -171,6 +177,24 @@ public interface EntryReader { String tag(); + /** Returns the known ID for {@link #tag()}, or {@code 0L} for an unknown tag. */ + long tagId(); + + /** + * This entry's tag name in the OpenTelemetry namespace: the rename the registry declares for + * it, else its Datadog name (pass-through, the default), else — for a custom tag, which the + * registry does not name at all — {@link #tag()} itself. + * + *

Never null, which is the point of asking the reader rather than the codec. {@link + * KnownTagCodec#openTelemetryTagOf} owns the naming policy but returns null for an unknown id, + * because only the holder of the entry knows the key to fall back to. This completes that one + * step and nothing more, so the policy still lives in exactly one place. + */ + default String openTelemetryTag() { + String otelTag = KnownTagCodec.openTelemetryTagOf(tagId()); + return otelTag != null ? otelTag : tag(); + } + byte type(); boolean is(byte type); @@ -342,7 +366,12 @@ static Entry newDoubleEntry(String tag, Double box) { volatile String strCache = null; private Entry(String tag, byte type, long prim, Object obj) { - super(tag); + /* + * Canonicalize known names at the single Entry construction point, so Datadog and + * OpenTelemetry spellings use the same TagMap key. This adds a StringIndex lookup to every + * new entry, including on the application thread. + */ + super(KnownTagCodec.canonicalTagName(tag)); this.lazyTagHash = 0; // lazily computed this.rawType = type; @@ -361,6 +390,15 @@ int hash() { return hash; } + @Override + public long tagId() { + /* + * Resolve on demand. Only OTLP serialization currently needs the ID, so caching it here + * would add a field to every Entry to save a lookup on exported tags. + */ + return KnownTagCodec.keyOf(this.tag); + } + @Override public Entry entry() { return this; @@ -914,12 +952,17 @@ public Ledger smartRemove(String tag) { } private boolean contains(String tag) { + // Entries and removals are both recorded under their canonical Datadog name (see newAnyEntry + // et al. and EntryRemoval's own constructor); canonicalize the query the same way or it + // would miss. + String canonicalTag = KnownTagCodec.canonicalTagName(tag); + EntryChange[] thisChanges = this.entryChanges; // min is to clamp, so bounds check elimination optimization works int lenClamp = Math.min(this.nextPos, thisChanges.length); for (int i = 0; i < lenClamp; ++i) { - if (thisChanges[i].matches(tag)) return true; + if (thisChanges[i].matches(canonicalTag)) return true; } return false; } @@ -928,13 +971,14 @@ private boolean contains(String tag) { * Just for testing */ Entry findLastEntry(String tag) { + String canonicalTag = KnownTagCodec.canonicalTagName(tag); EntryChange[] thisChanges = this.entryChanges; // min is to clamp, so ArrayBoundsCheckElimination optimization works int clampLen = Math.min(this.nextPos, thisChanges.length) - 1; for (int i = clampLen; i >= 0; --i) { EntryChange thisChange = thisChanges[i]; - if (!thisChange.isRemoval() && thisChange.matches(tag)) return (Entry) thisChange; + if (!thisChange.isRemoval() && thisChange.matches(canonicalTag)) return (Entry) thisChange; } return null; } @@ -1254,7 +1298,12 @@ public Set> entrySet() { } public Entry getEntry(String tag) { - Entry local = this.getLocalEntry(tag); + // Entries are stored under their canonical Datadog name (see Entry's constructor); a lookup by + // an OpenTelemetry rename must canonicalize the same way, or it would hash to the wrong bucket + // and silently miss the entry stored under the Datadog name. + String canonicalTag = KnownTagCodec.canonicalTagName(tag); + + Entry local = this.getLocalEntry(canonicalTag); if (local != null) { // Local entry shadows the parent (local-wins) — unchanged hot path. return local; @@ -1266,10 +1315,10 @@ public Entry getEntry(String tag) { if (parent == null) { return null; } - if (this.removedFromParent != null && this.removedFromParent.contains(tag)) { + if (this.removedFromParent != null && this.removedFromParent.contains(canonicalTag)) { return null; // tombstoned: removed locally, do not read through } - return parent.getEntry(tag); + return parent.getEntry(canonicalTag); } /** Looks up an entry in this map's own buckets only — no read-through to the parent. */ @@ -1734,7 +1783,11 @@ public boolean remove(String tag) { public Entry getAndRemove(String tag) { this.checkWriteAccess(); - Entry localRemoved = this.removeLocal(tag); + // See getEntry: entries are stored under their canonical Datadog name, so a removal by an + // OpenTelemetry rename must canonicalize first to find (and tombstone) the right entry. + String canonicalTag = KnownTagCodec.canonicalTagName(tag); + + Entry localRemoved = this.removeLocal(canonicalTag); TagMap parent = this.parent; if (parent != null) { @@ -1743,16 +1796,16 @@ public Entry getAndRemove(String tag) { // local entry if there was one, otherwise the parent's (which we now hide). Single-parent in // phase 1; rare path (only when removing a parent-exposed key). boolean alreadyTombstoned = - this.removedFromParent != null && this.removedFromParent.contains(tag); + this.removedFromParent != null && this.removedFromParent.contains(canonicalTag); if (!alreadyTombstoned) { - Entry parentEntry = parent.getEntry(tag); + Entry parentEntry = parent.getEntry(canonicalTag); if (parentEntry != null) { if (this.removedFromParent == null) { // Small initial capacity: this set is rare and almost always holds only a handful of // tombstoned keys, so the default 16-bucket HashSet table would be wasteful. this.removedFromParent = new HashSet<>(4); } - this.removedFromParent.add(tag); + this.removedFromParent.add(canonicalTag); return localRemoved != null ? localRemoved : parentEntry; } } @@ -2855,17 +2908,20 @@ final class EntryReadingHelper implements TagMap.EntryReader { private Map.Entry mapEntry; private String tag; private Object value; + private long tagId; void set(String tag, Object value) { this.mapEntry = null; this.tag = tag; this.value = value; + this.tagId = TagMap.TAG_ID_NOT_COMPUTED; // resolve lazily via keyOf on first tagId() access } void set(Map.Entry mapEntry) { this.mapEntry = mapEntry; this.tag = mapEntry.getKey(); this.value = mapEntry.getValue(); + this.tagId = TagMap.TAG_ID_NOT_COMPUTED; // resolve lazily via keyOf on first tagId() access } @Override @@ -2873,6 +2929,16 @@ public String tag() { return this.tag; } + @Override + public long tagId() { + long id = this.tagId; + if (id != TagMap.TAG_ID_NOT_COMPUTED) return id; + + id = KnownTagCodec.keyOf(this.tag); + this.tagId = id; + return id; + } + @Override public byte type() { return TagValueConversions.typeOf(this.value); diff --git a/internal-api/src/test/java/datadog/trace/api/EntryReadingHelperTest.java b/internal-api/src/test/java/datadog/trace/api/EntryReadingHelperTest.java index c47f15e1b3e..5da8564cb87 100644 --- a/internal-api/src/test/java/datadog/trace/api/EntryReadingHelperTest.java +++ b/internal-api/src/test/java/datadog/trace/api/EntryReadingHelperTest.java @@ -136,4 +136,34 @@ void setMapEntryCreatesEntry() { assertEquals("k", entry.tag()); assertEquals(99, entry.objectValue()); } + + @Test + void tagIdResolvesLazilyAndCaches() { + EntryReadingHelper helper = new EntryReadingHelper(); + helper.set("http.method", "GET"); + + long expectedId = KnownTagCodec.keyOf("http.method"); + assertEquals(expectedId, helper.tagId()); + // second call hits the already-resolved cache rather than re-resolving + assertEquals(expectedId, helper.tagId()); + } + + @Test + void tagIdOfUnknownTagResolvesToZeroAndCaches() { + EntryReadingHelper helper = new EntryReadingHelper(); + helper.set("my.custom.tag", "value"); + + assertEquals(0L, helper.tagId()); + assertEquals(0L, helper.tagId()); + } + + @Test + void tagIdIsRecomputedAfterSet() { + EntryReadingHelper helper = new EntryReadingHelper(); + helper.set("http.method", "GET"); + assertEquals(KnownTagCodec.keyOf("http.method"), helper.tagId()); + + helper.set(new AbstractMap.SimpleEntry<>("db.type", "mysql")); + assertEquals(KnownTagCodec.keyOf("db.type"), helper.tagId()); + } } diff --git a/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java new file mode 100644 index 00000000000..ecd9bb61e50 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java @@ -0,0 +1,227 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Checks the generated registry's name resolution and tag ID encoding. Datadog and OpenTelemetry + * names resolve to the same ID, while output names remain namespace-specific. + */ +class KnownTagsTest { + + /** (name, id) pairs across the groups — keyOf returns the id verbatim. */ + static Stream knownTags() { + return Stream.of( + Arguments.of(DDTags.PARENT_ID, KnownTags.DD_PARENT_ID), + Arguments.of(DDTags.BASE_SERVICE, KnownTags.DD_BASE_SERVICE_ID), + Arguments.of(Tags.VERSION, KnownTags.VERSION_ID), + Arguments.of("env", KnownTags.ENV_ID), + Arguments.of(DDTags.DJM_ENABLED, KnownTags.DD_DJM_ENABLED_ID), + Arguments.of(DDTags.DSM_ENABLED, KnownTags.DD_DSM_ENABLED_ID), + Arguments.of(DDTags.TRACER_HOST, KnownTags.DD_TRACER_HOST_ID), + Arguments.of(DDTags.DD_INTEGRATION, KnownTags.DD_INTEGRATION_ID), + Arguments.of(DDTags.DD_SVC_SRC, KnownTags.DD_SVC_SRC_ID), + Arguments.of(Tags.PEER_SERVICE, KnownTags.PEER_SERVICE_ID), + Arguments.of(DDTags.PEER_SERVICE_REMAPPED_FROM, KnownTags.DD_PEER_SERVICE_REMAPPED_FROM_ID), + Arguments.of(Tags.HTTP_METHOD, KnownTags.HTTP_METHOD_ID), + Arguments.of(Tags.HTTP_ROUTE, KnownTags.HTTP_ROUTE_ID), + Arguments.of(Tags.HTTP_URL, KnownTags.HTTP_URL_ID), + Arguments.of(Tags.PEER_HOSTNAME, KnownTags.PEER_HOSTNAME_ID), + Arguments.of(Tags.PEER_HOST_IPV4, KnownTags.PEER_IPV4_ID), + Arguments.of(Tags.PEER_HOST_IPV6, KnownTags.PEER_IPV6_ID), + Arguments.of(Tags.PEER_PORT, KnownTags.PEER_PORT_ID), + Arguments.of(Tags.COMPONENT, KnownTags.COMPONENT_ID), + Arguments.of(Tags.SPAN_KIND, KnownTags.SPAN_KIND_ID), + Arguments.of(DDTags.LANGUAGE_TAG_KEY, KnownTags.LANGUAGE_ID), + Arguments.of(Tags.DB_TYPE, KnownTags.DB_TYPE_ID), + Arguments.of(Tags.DB_INSTANCE, KnownTags.DB_INSTANCE_ID), + Arguments.of(Tags.DB_USER, KnownTags.DB_USER_ID), + Arguments.of(Tags.DB_OPERATION, KnownTags.DB_OPERATION_ID), + Arguments.of(Tags.DB_POOL_NAME, KnownTags.DB_POOL_NAME_ID)); + } + + /** + * (otelName, canonicalId, datadogName) — the OpenTelemetry name resolves (keyOf) to the canonical + * tag's id; datadogNameOf returns the Datadog name and openTelemetryNameOf returns the OTel name. + */ + static Stream otelNamedTags() { + return Stream.of( + Arguments.of("http.request.method", KnownTags.HTTP_METHOD_ID, "http.method"), + Arguments.of( + "http.response.status_code", KnownTags.HTTP_STATUS_CODE_ID, "http.status_code"), + Arguments.of("url.full", KnownTags.HTTP_URL_ID, "http.url"), + Arguments.of("user_agent.original", KnownTags.HTTP_USERAGENT_ID, "http.useragent"), + Arguments.of("client.address", KnownTags.HTTP_CLIENT_IP_ID, "http.client_ip"), + Arguments.of("db.system", KnownTags.DB_TYPE_ID, "db.type"), + Arguments.of("db.operation.name", KnownTags.DB_OPERATION_ID, "db.operation"), + Arguments.of("db.query.text", KnownTags.DB_STATEMENT_ID, "db.statement"), + Arguments.of("service.name", KnownTags.SERVICE_ID, "service")); + } + + /** + * Trace-level tags (live on the TraceSegment's TagMap) — their id carries the LEVEL_TRACE bit. + */ + static Stream traceLevelTags() { + return Stream.of( + Arguments.of(KnownTags.DD_BASE_SERVICE_ID), + Arguments.of(KnownTags.VERSION_ID), + Arguments.of(KnownTags.ENV_ID), + Arguments.of(KnownTags.LANGUAGE_ID), + Arguments.of(KnownTags.RUNTIME_ID), + Arguments.of(KnownTags.DD_TRACER_HOST_ID), + Arguments.of(KnownTags.DD_DJM_ENABLED_ID)); + } + + /** Span-level tags — their id leaves the LEVEL_TRACE bit clear. */ + static Stream spanLevelTags() { + return Stream.of( + Arguments.of(KnownTags.HTTP_METHOD_ID), + Arguments.of(KnownTags.HTTP_URL_ID), + Arguments.of(KnownTags.DB_TYPE_ID), + Arguments.of(KnownTags.COMPONENT_ID), + Arguments.of(KnownTags.SPAN_KIND_ID), + Arguments.of(KnownTags.PEER_SERVICE_ID)); + } + + @ParameterizedTest + @MethodSource("knownTags") + void keyOfResolvesNameToId(String name, long id) { + assertEquals(id, KnownTagCodec.keyOf(name), "keyOf(" + name + ")"); + } + + @ParameterizedTest + @MethodSource("knownTags") + void nameOfResolvesIdToName(String name, long id) { + assertEquals(name, KnownTagCodec.nameOf(id), "nameOf(" + name + ")"); + } + + @ParameterizedTest + @MethodSource("otelNamedTags") + void otelNameResolvesToCanonicalId(String otelName, long id, String datadogName) { + // Inbound (keyOf) is many->one: both names land on the same canonical id. + assertEquals(id, KnownTagCodec.keyOf(otelName), "keyOf(" + otelName + ")"); + assertEquals(id, KnownTagCodec.keyOf(datadogName), "keyOf(" + datadogName + ")"); + } + + @ParameterizedTest + @MethodSource("otelNamedTags") + void namespaceAccessorsReturnPerNamespaceName(String otelName, long id, String datadogName) { + assertEquals(datadogName, KnownTagCodec.datadogNameOf(id), "datadogNameOf"); + assertEquals(otelName, KnownTagCodec.openTelemetryNameOf(id), "openTelemetryNameOf"); + // nameOf stays the Datadog name -- outbound is namespace-specific, not normalized to OTel. + assertEquals(datadogName, KnownTagCodec.nameOf(id), "nameOf stays Datadog"); + } + + @ParameterizedTest + @MethodSource("otelNamedTags") + void openTelemetryTagOfReturnsTheRename(String otelName, long id, String datadogName) { + assertEquals(otelName, KnownTagCodec.openTelemetryTagOf(id), "openTelemetryTagOf"); + } + + @Test + void openTelemetryTagOfPassesThroughWhenThereIsNoRename() { + // http.route declares no otel-name, so the OpenTelemetry namespace emits the Datadog name. + assertNull(KnownTagCodec.openTelemetryNameOf(KnownTags.HTTP_ROUTE_ID), "no declared rename"); + assertEquals( + KnownTagCodec.nameOf(KnownTags.HTTP_ROUTE_ID), + KnownTagCodec.openTelemetryTagOf(KnownTags.HTTP_ROUTE_ID), + "pass-through falls back to the Datadog name"); + } + + @Test + void openTelemetryTagOfIsNullForAnUnknownId() { + // A custom tag has no registry name in any namespace; only its holder knows its key. + assertNull(KnownTagCodec.openTelemetryTagOf(0L)); + } + + @Test + void tagsWithoutOtelNameReturnNull() { + assertNull(KnownTagCodec.openTelemetryNameOf(KnownTags.HTTP_ROUTE_ID)); // no OTel name declared + assertNull(KnownTagCodec.openTelemetryNameOf(0L)); // unknown id + } + + @Test + void unknownNamesResolveToZero() { + assertEquals(0L, KnownTagCodec.keyOf("definitely.not.a.known.tag")); + assertEquals(0L, KnownTagCodec.keyOf("http.statuscode")); // close-but-not-listed + assertEquals(0L, KnownTagCodec.keyOf("")); + } + + @Test + void unknownIdsResolveToNullName() { + assertNull(KnownTagCodec.nameOf(0L)); + assertNull(KnownTagCodec.nameOf(KnownTagCodec.makeTagId(9999))); // serial with no assigned tag + } + + /** + * A mixin declares tags for the span types its {@code applies:} names. {@code ci_visibility} + * applies to {@code test}, which the conventions do not model yet -- so these tags belong to no + * concrete type's resolved set. They must still be registered: an id is identity, and identity + * does not depend on layout. Building the registry by resolving concrete types instead dropped + * all four silently, leaving keyOf to report live CI Visibility tags as unknown. + */ + @ParameterizedTest + @MethodSource("declarationOnlyMixinTags") + void mixinTagsAreRegisteredEvenWhenTheirSpanTypeIsNotModeled(String name, long id) { + assertEquals(id, KnownTagCodec.keyOf(name), "keyOf(" + name + ")"); + assertEquals(name, KnownTagCodec.nameOf(id), "nameOf(" + name + ")"); + } + + /** Tags reachable only through a mixin whose {@code applies:} target is not modeled. */ + static Stream declarationOnlyMixinTags() { + return Stream.of( + Arguments.of(KnownTags.TEST_NAME, KnownTags.TEST_NAME_ID), + Arguments.of(KnownTags.TEST_SUITE_NAME, KnownTags.TEST_SUITE_ID), + Arguments.of(KnownTags.TEST_STATUS_NAME, KnownTags.TEST_STATUS_ID), + Arguments.of(KnownTags.TEST_FRAMEWORK_NAME, KnownTags.TEST_FRAMEWORK_ID)); + } + + @Test + void globalSerialsAreUnique() { + List serials = new ArrayList<>(); + knownTags().forEach(a -> serials.add((long) KnownTagCodec.serialNum((Long) a.get()[1]))); + assertEquals(serials.size(), new HashSet<>(serials).size(), "globalSerials must be unique"); + } + + @ParameterizedTest + @MethodSource("traceLevelTags") + void traceLevelTagsCarryLevelBit(long id) { + assertTrue(KnownTagCodec.isTraceLevel(id), "isTraceLevel"); + } + + @ParameterizedTest + @MethodSource("spanLevelTags") + void spanLevelTagsClearLevelBit(long id) { + assertFalse(KnownTagCodec.isTraceLevel(id), "not trace-level"); + } + + @Test + void levelBitCompositionRoundTrips() { + long spanId = KnownTagCodec.makeTagId(300); // no level bit + assertFalse(KnownTagCodec.isTraceLevel(spanId)); + long traceId = KnownTagCodec.traceLevel(spanId); + assertTrue(KnownTagCodec.isTraceLevel(traceId)); + // level bit is orthogonal to the serial — it survives setting the bit + assertEquals(KnownTagCodec.serialNum(spanId), KnownTagCodec.serialNum(traceId)); + assertEquals(traceId, KnownTagCodec.traceLevel(traceId), "traceLevel is idempotent"); + } + + @Test + void serialEncodingRoundTrips() { + long id = KnownTagCodec.makeTagId(263); + assertEquals(263, KnownTagCodec.serialNum(id)); + assertFalse(KnownTagCodec.isTraceLevel(id)); + } +} diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapNamespaceNamesTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapNamespaceNamesTest.java new file mode 100644 index 00000000000..1776f32c291 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/TagMapNamespaceNamesTest.java @@ -0,0 +1,81 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * {@link TagMap.EntryReader#openTelemetryTag()} — the reader's view of its tag in the OpenTelemetry + * namespace. {@link KnownTagCodec#openTelemetryTagOf} owns the naming and returns null for a tag it + * does not know; the reader completes that by falling back to its own key, which is the one thing + * the codec cannot do. + */ +class TagMapNamespaceNamesTest { + + @Test + void renamedTagReadsUnderItsOpenTelemetryName() { + assertEquals("http.request.method", otelTagOf("http.method", "GET")); + } + + @Test + void tagWithoutARenamePassesThroughUnderItsDatadogName() { + // http.route declares no otel-name, so the OpenTelemetry namespace keeps the Datadog spelling. + assertEquals("http.route", otelTagOf("http.route", "/orders/:id")); + } + + @Test + void customTagFallsBackToItsOwnKey() { + // Not in the registry at all: the codec has no name for it, so the reader supplies its key. + assertEquals("my.app.tenant", otelTagOf("my.app.tenant", "acme")); + } + + @Test + void anOpenTelemetrySpellingNormalizesToTheOneName() { + // keyOf is many->one, so an entry written under the OTel name resolves to the same tag and + // reads back under that name -- not under two different ones depending on how it was written. + assertEquals("http.request.method", otelTagOf("http.request.method", "POST")); + assertEquals( + otelTagOf("http.method", "GET"), + otelTagOf("http.request.method", "POST"), + "both spellings of one tag must read under the same OpenTelemetry name"); + } + + @Test + void openTelemetryNameDiffersFromTheDatadogNameForARenamedTag() { + TagMap map = TagMap.create(); + map.set("http.method", "GET"); + TagMap.EntryReader reader = readerFor(map, "http.method"); + assertEquals("http.method", reader.tag(), "tag() stays the key as written"); + assertNotEquals( + reader.tag(), reader.openTelemetryTag(), "a rename must actually change the emitted name"); + } + + private static String otelTagOf(String tag, Object value) { + TagMap map = TagMap.create(); + map.set(tag, value); + return readerFor(map, tag).openTelemetryTag(); + } + + /** + * The entry for {@code tag}, having first checked that iteration agrees with it — the iterator + * may hand out a reused flyweight rather than the entry itself, so the two paths are worth + * pinning together. + */ + private static TagMap.EntryReader readerFor(TagMap map, String tag) { + Map otelByTag = new HashMap<>(); + map.forEach(reader -> otelByTag.put(reader.tag(), reader.openTelemetryTag())); + + TagMap.Entry entry = map.getEntry(tag); + // Key off the entry's own (canonical) tag(), not the possibly-pre-normalization `tag` + // argument: entries are stored under their canonical Datadog name, so a lookup by an + // OpenTelemetry alias returns an entry whose tag() differs from the string used to set it. + assertEquals( + otelByTag.get(entry.tag()), + entry.openTelemetryTag(), + "iteration and getEntry must agree on the OpenTelemetry name for " + tag); + return entry; + } +} diff --git a/tag-conventions.yaml b/tag-conventions.yaml new file mode 100644 index 00000000000..f35498e980b --- /dev/null +++ b/tag-conventions.yaml @@ -0,0 +1,169 @@ +# Tag conventions — LANGUAGE-AGNOSTIC domain spec (structure + semantics only) +# --------------------------------------------------------------------------- +# The code generator consumes THIS file to emit each language's tag-id constants and its +# id<->name resolver. A tag id is IDENTITY (a globally unique serial + the trace-level bit); +# storage layout (the dense store's slot assignment) and set-path routing (which keys the tracer +# intercepts into span fields or sampling directives) are per-language concerns that arrive with +# the code that consumes them, via a per-language overlay alongside this file. +# +# TRACE-LEVEL is its own thing (its own TagMap "type" on the TraceSegment) — the process/trace +# constants + product flags that are set once per trace, NOT per span. Declared explicitly in the +# `trace_level` section below (a distinct tier), never inferred from `source`. +# +# SPAN TYPES compose three ways: +# extends — structural is-a inheritance (http.server is-a http is-a base). `base` is implicitly +# in every span; abstract layers exist only to be extended. +# include — a span type PULLS in a mixin it intrinsically has (has-a; core-owned). +# applies — a mixin PUSHES itself onto span types, gated by `enabled_by`. +# resolved_tags(type) = own + extends-chain (incl base) + included mixins + applied mixins (de-duped). +# +# tag fields (DOMAIN only): dd-name | type (string|int|long|boolean|double) +# | required (required|conditional|recommended|optional|opt_in) | otel-name. +# dd-name is the canonical Datadog-namespace name AND the tag's identity. otel-name is OPTIONAL and +# tri-state: +# - absent => the OpenTelemetry name is IMPLICITLY the dd-name (the tag passes through under +# its Datadog name; this is the RFC "retain" default for tags with no rename). +# - a name => rename: the tag is emitted under that OpenTelemetry-namespace name instead. +# - the literal none => Datadog-only: the tag has NO OpenTelemetry name (suppressed from OTel). This +# value is reserved — no tag uses it today (the RFC renames or retains, never +# suppresses), and real suppression is a follow-on; it currently behaves as +# pass-through. +# A tag is one identity across span types/mixins, so it is DECLARED exactly once — a second declaration +# fails the build. Put a tag shared by several span types on their common parent or a mixin; any other +# span type that carries it uses `{ ref: , required: }`, which may override only +# `required` (type and otel-name come from the one declaration). A span-kind-dependent mapping is a +# derivation, not a rename. +# An otel-name is canonicalized to its tag on EVERY span kind, so a rename declared on a concrete span +# type must also set `span-kind-neutral: true`: the author's assertion that the OpenTelemetry attribute +# means this tag wherever OTel uses it. (Renames in a shared scope -- an abstract parent or a mixin -- +# need no flag.) OTel's server.address and network.peer.address fail that test: on a client span they +# name the remote server. +# The id coordinate (group-decl / field-decl) is NOT authored here — the generator assigns it: each +# declaration source (the trace-level tier, each span type, each mixin) is a group, and within a +# group `field-decl` numbers the dense (required/conditional/recommended) tags; the rest are +# bucketed. See the design doc. +# --------------------------------------------------------------------------- + +# Trace-level tier: its own TagMap on the TraceSegment. Set once per trace, not per span. +# (Their OTel mapping is a resource-attribute follow-on; they pass through under dd-name for now.) +trace_level: + tags: + - { dd-name: _dd.base_service, type: string, required: required } + - { dd-name: version, type: string, required: recommended } + - { dd-name: env, type: string, required: recommended } + - { dd-name: language, type: string, required: required } + - { dd-name: runtime-id, type: string, required: required } + - { dd-name: _dd.tracer_host, type: string, required: recommended } + - { dd-name: _dd.git.commit.sha, type: string, required: recommended } + - { dd-name: _dd.git.repository_url, type: string, required: recommended } + # product .enabled flags — process-constant; present on the trace segment regardless of whether + # the product is enabled (the flag carries the state), so always-present => recommended. + - { dd-name: _dd.profiling.enabled, type: boolean, required: recommended } + - { dd-name: _dd.dsm.enabled, type: boolean, required: recommended } + - { dd-name: _dd.appsec.enabled, type: boolean, required: recommended } + - { dd-name: _dd.djm.enabled, type: boolean, required: recommended } + - { dd-name: _dd.civisibility.enabled, type: boolean, required: recommended } + +span_types: + # root: per-span tags every span has (incl. the per-span core tags parent_id / integration / svc_src + # — core-set but per-span, so NOT trace-level). + base: + abstract: true + tags: + - { dd-name: _dd.parent_id, type: string, required: required } + - { dd-name: service, type: string, required: required, otel-name: service.name } + - { dd-name: component, type: string, required: required } + - { dd-name: span.kind, type: string, required: required } # OTel span kind is a first-class field, not an attribute + - { dd-name: _dd.integration, type: string, required: recommended } + - { dd-name: _dd.svc_src, type: string, required: optional } + - { dd-name: error.type, type: string, required: recommended } # TODO(otel): map error.* to exception.* semconv + - { dd-name: error.message, type: string, required: recommended } + - { dd-name: error.stack, type: string, required: recommended } + + http: + abstract: true + extends: base + tags: + - { dd-name: http.method, type: string, required: required, otel-name: http.request.method } + - { dd-name: http.status_code, type: int, required: conditional, otel-name: http.response.status_code } + - { dd-name: network.protocol.version, type: string, required: recommended } # passes through: dd-name already is the OTel name + - { dd-name: http.url, type: string, required: required, otel-name: url.full } # shared by http.server and http.client => one otel-name. url.full is the client-correct rename; server's spec mapping (url.path + url.scheme + url.query) is a one-to-many split reserved for the derivation layer (needs span.kind). TODO(otel): server split. + + http.server: + extends: http + tags: + - { dd-name: http.route, type: string, required: conditional } # passes through: dd-name already is the OTel name + - { dd-name: http.hostname, type: string, required: required } # not a rename: OTel server.address is the remote server on client spans; needs the span-kind-aware derivation layer + - { dd-name: http.useragent, type: string, required: recommended, otel-name: user_agent.original, span-kind-neutral: true } + - { dd-name: http.query.string, type: string, required: recommended } # not a rename: url.query is also carried inside url.full, while http.url excludes the query (QueryObfuscator re-appends it) + - { dd-name: servlet.path, type: string, required: optional } + - { dd-name: servlet.context, type: string, required: optional } + - { dd-name: http.client_ip, type: string, required: recommended, otel-name: client.address, span-kind-neutral: true } + - { dd-name: network.client.ip, type: string, required: recommended } # not a rename: OTel network.peer.address is the remote end on every span kind (the server on client spans) + + http.client: + extends: http + include: [ peer ] + tags: + - { dd-name: http.resend_count, type: int, required: recommended } + + db.client: + extends: base + include: [ peer ] + tags: + - { dd-name: db.type, type: string, required: required, otel-name: db.system, span-kind-neutral: true } + - { dd-name: db.instance, type: string, required: recommended } # TODO(otel): db.namespace + - { dd-name: db.operation, type: string, required: recommended, otel-name: db.operation.name, span-kind-neutral: true } + - { dd-name: db.user, type: string, required: recommended } + - { dd-name: db.pool.name, type: string, required: optional } + - { dd-name: db.statement, type: string, required: recommended, otel-name: db.query.text, span-kind-neutral: true } + + view.render: + extends: base + tags: + - { dd-name: view.name, type: string, required: recommended } + +mixins: + # peer — outbound/remote-peer capability, PULLED via `include` by client span types. + peer: + tags: + - { dd-name: peer.service, type: string, required: recommended } + - { dd-name: _dd.peer.service.source, type: string, required: recommended } + - { dd-name: _dd.peer.service.remapped_from, type: string, required: recommended } + - { dd-name: peer.hostname, type: string, required: recommended } + - { dd-name: peer.ipv4, type: string } + - { dd-name: peer.ipv6, type: string } + - { dd-name: peer.port, type: int } + + # ci_visibility — per-span test tags. Its capability flag (_dd.civisibility.enabled) lives in + # trace_level, outside this mixin (general rule: capability flags are trace-level, mixins hold the + # per-span tags). Applies to the `test` span type, which is not modeled here yet — so these tags + # get ids (identity does not depend on layout) but contribute to no type's resolved set until it + # is. The generator reports that gap in resolved-tags.txt. + ci_visibility: + enabled_by: dd.civisibility.enabled + applies: [ test ] + tags: + - { dd-name: test.name, type: string, required: recommended } + - { dd-name: test.suite, type: string, required: recommended } + - { dd-name: test.status, type: string, required: recommended } + - { dd-name: test.framework, type: string, required: recommended } + +# --------------------------------------------------------------------------- +# Notes +# - Product .enabled flags moved to `trace_level` (process-constant) — the old product mixins held +# only those flags, so they dissolved. `enabled_by`/attachment gating is a runtime concern. +# - span.kind enumerates: server | client | producer | consumer | internal | broker. +# - Some keys (resource.name, error, sampling.priority, ...) are accepted by setTag but routed to a +# span field or a trace directive instead of tag storage. That routing is a per-language tracer +# concern, so it is NOT modelled here; such a key appears above only when it also needs an id and +# a name (service does, for OpenTelemetry's service.name). +# - Tags with no otel-name pass through under their Datadog name (RFC "retain"). A `# TODO(otel)` note +# marks a pending OpenTelemetry-team review of a mapping that is not yet a settled rename. +# - db.statement/db.query.text is deliberately never re-exposed as a stored tag once consumed into +# resource.name, regardless of which alias set it -- the two names are equivalent for identity/id +# purposes, but the raw value (often unobfuscated SQL) is intentionally not retained. This is an +# open question (not yet resolved one way or the other): revisit only as a language-agnostic policy +# change (i.e. decide to start storing/exporting it in every tracer), not as a per-language or +# per-spelling special case. +# ---------------------------------------------------------------------------