diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 53f77a129b4..bf1ce358fdf 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,6 +12,7 @@ /README.md @DataDog/apm-java /SECURITY.md @DataDog/apm-java /SUPPORT.md @DataDog/apm-java +/tag-conventions.yaml @DataDog/apm-java @DataDog/apm-lang-platform-java @DataDog/apm-sdk-capabilities-java /dd-smoke-tests/README.md @DataDog/apm-java /docs/ @DataDog/apm-java diff --git a/build-logic/settings.gradle.kts b/build-logic/settings.gradle.kts index db1b1d5d1c8..a0dc4aa5b1e 100644 --- a/build-logic/settings.gradle.kts +++ b/build-logic/settings.gradle.kts @@ -55,6 +55,9 @@ dependencyResolutionManagement { rootProject.name = "build-logic" -include(":conventions") -include(":smoke-test") +include( + ":conventions", + ":smoke-test", + ":tag-registry", +) include(":testcontainers") diff --git a/build-logic/tag-registry/build.gradle.kts b/build-logic/tag-registry/build.gradle.kts new file mode 100644 index 00000000000..74029465ae5 --- /dev/null +++ b/build-logic/tag-registry/build.gradle.kts @@ -0,0 +1,62 @@ +plugins { + `java-gradle-plugin` + `kotlin-dsl` + `jvm-test-suite` + alias(libs.plugins.spotless) +} + +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} + +kotlin { + compilerOptions { + jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17) + } +} + +dependencies { + // Keep aligned with buildSrc, whose Jackson classes are visible to applied plugins. + implementation(platform("com.fasterxml.jackson:jackson-bom:2.17.2")) + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") +} + +gradlePlugin { + plugins { + create("tag-registry-generator") { + id = "dd-trace-java.tag-registry-generator" + implementationClass = "datadog.buildlogic.tagRegistry.TagRegistryGeneratorPlugin" + } + } +} + +testing { + suites { + named("test") { + useJUnitJupiter(libs.versions.junit5) + dependencies { + implementation(platform("org.junit:junit-bom:${libs.versions.junit5.get()}")) + implementation(libs.assertj.core) + implementation(libs.tabletest) + implementation(gradleTestKit()) + } + } + } +} + +spotless { + kotlin { + target("src/**/*.kt") + ktlint(libs.versions.ktlint.get()).editorConfigOverride( + mapOf( + "indent_size" to "2", + "ktlint_standard_trailing-comma-on-call-site" to "disabled", + "ktlint_standard_trailing-comma-on-declaration-site" to "disabled", + ), + ) + } + kotlinGradle { + ktlint(libs.versions.ktlint.get()).editorConfigOverride(mapOf("indent_size" to "2")) + } +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/GenerateKnownTagsTask.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/GenerateKnownTagsTask.kt new file mode 100644 index 00000000000..bd567a67854 --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/GenerateKnownTagsTask.kt @@ -0,0 +1,31 @@ +package datadog.buildlogic.tagRegistry + +import org.gradle.api.DefaultTask +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction + +/** + * Generates KnownTags.java and assignment reports from tag-conventions.yaml. + * Declares the inputs and outputs for Gradle's up-to-date checks and build cache. + */ +@CacheableTask +abstract class GenerateKnownTagsTask : DefaultTask() { + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + abstract val tagConventionsFile: RegularFileProperty + + @get:OutputDirectory abstract val destinationDirectory: DirectoryProperty + + @TaskAction + fun generate() { + val outDir = destinationDirectory.get().asFile + TagRegistryGenerator.generate(tagConventionsFile.get().asFile, outDir) + logger.lifecycle("tag-registry: generated -> $outDir") + } +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt new file mode 100644 index 00000000000..0b59976fc4d --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt @@ -0,0 +1,221 @@ +package datadog.buildlogic.tagRegistry + +import com.fasterxml.jackson.core.io.JsonStringEncoder +import java.util.Locale + +/** + * Emits the generated `KnownTags.java` from a [TagRegistry]. Public API first — per-tag + * `_NAME` (string) + `_ID` (encoded long, literal) couplets with a trailing `// makeTagId(...)` + * derivation comment — then the package-private `_SERIAL_NUM` constants, the + * `StringIndex.EmbeddingSupport` keyOf table and the resolver's name switches. + */ +object KnownTagsEmitter { + + fun emit(reg: TagRegistry, pkg: String, className: String): String { + // Sanitize a tag name into a Java identifier base (not yet unique). + fun sanitize(name: String): String { + var c = name.uppercase().replace(Regex("[^A-Za-z0-9]"), "_").replace(Regex("_+"), "_").trim('_') + if (c.isEmpty() || c[0].isDigit()) c = "T_$c" + return c + } + + // Collapse a duplicated trailing token so e.g. "resource.name" yields NAME (not NAME_NAME) and + // "_dd.parent_id" yields ID (not ID_ID); the non-duplicating pairs (ID + _NAME -> ID_NAME, NAME + // + _ID -> NAME_ID) are kept as-is. + fun withSuffix(base: String, suffix: String) = if (base.endsWith(suffix)) base else "$base$suffix" + + // NAME/ID/SERIAL_NUM constants are all fields of the same generated class, so uniqueness must be + // enforced on these FINAL suffixed identifiers, not on the pre-suffix base: two different base + // names can collapse to the same final identifier once a suffix is appended -- e.g. base + // "RESOURCE" suffixed with "_NAME" collides with a base that is already "RESOURCE_NAME" (which + // withSuffix leaves untouched, since it already ends with "_NAME"). + val used = HashSet() + fun unique(name: String): String { + var u = name + var n = 2 + while (!used.add(u)) { + u = "${name}_${n++}" + } + return u + } + + val nameOfConst = HashMap() + val idOfConst = HashMap() + val serialOfConst = HashMap() + val otelNameOfConst = HashMap() + for (t in reg.tags) { + val base = sanitize(t.name) + nameOfConst[t.name] = unique(withSuffix(base, "_NAME")) + idOfConst[t.name] = unique(withSuffix(base, "_ID")) + serialOfConst[t.name] = unique(withSuffix(base, "_SERIAL_NUM")) + // Suffix the pre-suffix base (not nameC), same as the other three: suffixing an + // already-suffixed identifier would produce a redundant compound like NAME_OTEL_NAME. + if (t.otelName != null) otelNameOfConst[t.name] = unique(withSuffix(base, "_OTEL_NAME")) + } + fun nameC(name: String) = nameOfConst.getValue(name) + fun idC(name: String) = idOfConst.getValue(name) + fun serialC(name: String) = serialOfConst.getValue(name) + fun otelNameC(name: String) = otelNameOfConst.getValue(name) + + val order = reg.tags.map { it.name } // stable emit order + // canonical name -> OpenTelemetry name, for the reverse (openTelemetryNameOf) switch. + val otelName = reg.tags.mapNotNull { t -> t.otelName?.let { t.name to it } }.toMap() + return buildString { + // Public API first (name + encoded id couplets), so readers see the useful parts up top; the + // serial ids and keyOf/resolver machinery follow below. Derivation is in the trailing comment. + appendLine( + """ + package $pkg; + + import datadog.trace.util.StringIndex; + + // GENERATED by the tag-registry code generator (dd-trace-java.tag-registry-generator). + // DO NOT EDIT. Source: tag-conventions.yaml. + public final class $className { + + // ---- tags ---- + """.trimIndent() + ) + + for (t in reg.tags) { + appendLine( + """ + public static final String ${nameC(t.name)} = "${escape(t.name)}"; + public static final long ${idC(t.name)} = ${hex(t.id)}; + """.trimIndent() + ) + if (t.otelName != null) { + appendLine(" public static final String ${otelNameC(t.name)} = \"${escape(t.otelName)}\";") + } + append("// makeTagId(serial=${t.serial})") + if (t.traceLevel) append(" + trace-level") + if (t.otelName != null) append(" -> ${escape(t.otelName)}") + appendLine(" <${escape(t.required)}>") + appendLine() + } + + // Serial numbers (globalSerial per tag) — package-private, consumed by the resolver switch. + appendLine(" // ---- serial numbers ----") + for (t in reg.tags) { + appendLine(" static final int ${serialC(t.name)} = ${t.serial};") + } + + // OpenTelemetry name -> canonical tag name. Validation ensures aliases are distinct from all + // canonical names. Sort by OTel name to keep output deterministic. + val otelByCanonical = + reg.tags + .mapNotNull { t -> t.otelName?.let { it to t.name } } + .sortedBy { it.first } + + // keyOf table (open-addressed, via StringIndex.EmbeddingSupport). Canonical names first, then + // OpenTelemetry names -- an OTel name resolves to its canonical tag's id (there is no distinct id + // for it), so keyOf(otelName) == keyOf(canonical); nameOf still returns the canonical name. + appendLine( + """ + + private static final String[] KEYOF_NAMES = { + """.trimIndent() + ) + order.forEach { appendLine(" ${nameC(it)},") } + otelByCanonical.forEach { (otel, _) -> + appendLine(" \"${escape(otel)}\",") + } + appendLine( + """ + }; + private static final long[] KEYOF_VALUES = { + """.trimIndent() + ) + order.forEach { appendLine(" ${idC(it)},") } + otelByCanonical.forEach { (_, canonical) -> + appendLine(" ${idC(canonical)},") + } + // Resolver. KnownTagCodec.Installed links to this field directly, so merely resolving a tag + // name initializes this class -- there is no registration step and no ordering to get wrong. + appendLine( + """ + }; + private static final int[] KEYOF_HASHES; + private static final String[] KEYOF_KEYS; + private static final long[] KEYOF_IDS; + + static { + StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES); + long[] ids = new long[data.names.length]; + for (int j = 0; j < KEYOF_NAMES.length; j++) { + ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = + KEYOF_VALUES[j]; + } + KEYOF_HASHES = data.hashes; + KEYOF_KEYS = data.names; + KEYOF_IDS = ids; + } + + /** + * The registry's name↔id tables, as a {@link KnownTagCodec.Resolver}. {@code KnownTagCodec} + * reads this field from its own holder, so the two classes complete each other: the codec owns + * the bit layout and the naming policy, this class owns the data. Nothing has to be called first. + */ + static final KnownTagCodec.Resolver RESOLVER = + new KnownTagCodec.Resolver() { + @Override + public String nameOf(long tagId) { + switch (KnownTagCodec.serialNum(tagId)) { + """.trimIndent() + ) + for (name in order) { + appendLine( + """ + case ${serialC(name)}: + return ${nameC(name)}; + """.trimIndent() + ) + } + // openTelemetryNameOf: canonical id -> OTel-namespace name, null when the tag has none. The + // caller (a serializer) owns any fall-back-to-Datadog-name policy; this stays a pure lookup. + appendLine( + """ + default: + return null; + } + } + + @Override + public String openTelemetryNameOf(long tagId) { + switch (KnownTagCodec.serialNum(tagId)) { + """.trimIndent() + ) + for (name in order) { + if (otelName[name] == null) continue + appendLine( + """ + case ${serialC(name)}: + return ${otelNameC(name)}; + """.trimIndent() + ) + } + appendLine( + """ + default: + return null; + } + } + + @Override + public long keyOf(String name) { + int slot = StringIndex.EmbeddingSupport.indexOf(KEYOF_HASHES, KEYOF_KEYS, name); + return slot < 0 ? 0L : KEYOF_IDS[slot]; + } + }; + + private $className() {} + } + """.trimIndent() + ) + } + } + + private fun hex(id: Long): String = "0x%016XL".format(Locale.ROOT, id) + + private fun escape(value: String): String = String(JsonStringEncoder.getInstance().quoteAsString(value)) +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt new file mode 100644 index 00000000000..75eaba1fe70 --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt @@ -0,0 +1,346 @@ +package datadog.buildlogic.tagRegistry + +/** + * Parses the tag conventions and resolves each span type's tags through `extends`, `include`, + * and `applies`. Declarations carry names, types, requirement levels, and optional OpenTelemetry + * renames. ID assignment and source generation are handled separately. + */ +class TagConventions private constructor( + private val spanTypes: Map, + private val mixins: Map, + private val traceLevel: List, +) { + /** One tag declaration: its Datadog name, type, requirement level, and optional rename. */ + data class Tag( + val name: String, + val type: String, + val required: String, + /** + * Explicit OpenTelemetry name from `otel-name`, or null when no rename is configured. + * When a rename is configured, both names resolve to the same tag ID. + * + * An omitted `otel-name` and the reserved literal `none` both produce null. Exporters + * currently use the Datadog name in either case; `none` does not suppress the tag yet. + */ + val otelName: String? = null, + /** + * Set `span-kind-neutral: true` for a rename declared on a concrete span type only + * when the Datadog and OpenTelemetry names denote the same value on every span kind. + * Name resolution ignores span kind, so the rename also applies outside that type. + * + * For example, `db.type` -> `db.system` on `db.client` qualifies: `db.system` only + * ever describes a database. Renames in `trace_level`, abstract types, and mixins need + * no flag. Setting this flag without a configured rename is invalid. + * + * The flag is interim: a follow-on replaces it with span-kind-aware name resolution. + */ + val spanKindNeutral: Boolean = false, + ) + + /** + * A `{ ref: , required: }` entry. It reuses a tag declared elsewhere, optionally + * at a different requirement level; its type and otel-name always come from that declaration. + */ + data class Ref(val name: String, val required: String?) + + data class SpanType( + val name: String, + val abstract: Boolean, + val extends: String?, + val include: List, + val tags: List, + val refs: List = emptyList(), + ) + + data class Mixin( + val name: String, + val appliesAll: Boolean, + val appliesTo: Set, + val tags: List, + val refs: List = emptyList(), + ) + + private val declarations: Map by lazy { + allDeclaredTags().associateBy { it.name } + } + + /** The tag a [Ref] names, at the ref's requirement level when it overrides one. */ + private fun materialize(ref: Ref): Tag { + val decl = declarations.getValue(ref.name) + return if (ref.required == null) decl else decl.copy(required = ref.required) + } + + /** Returns the names of non-abstract span types in sorted order. */ + fun concreteTypes(): List = spanTypes.values.filter { !it.abstract }.map { it.name }.sorted() + + /** + * Resolves tags from the root ancestor to the requested type, then adds mixins whose + * `applies` matches the type or an ancestor. + * + * Each type contributes its declarations, included mixins, then references. Duplicate + * names keep their first position; references can override requirement levels. References + * from `applies` mixins add only missing tags. + */ + fun resolve(typeName: String): List { + val result = LinkedHashMap() + fun add(t: Tag) = result.putIfAbsent(t.name, t) + fun applyRef(r: Ref) { + val current = result[r.name] + // Re-putting an existing key keeps its LinkedHashMap position. + result[r.name] = + when { + current == null -> materialize(r) + r.required == null -> current + else -> current.copy(required = r.required) + } + } + + val chain = ArrayList() + var cur: SpanType? = spanTypes[typeName] + while (cur != null) { + chain.add(cur) + cur = cur.extends?.let { spanTypes[it] } + } + for (st in chain.asReversed()) { + st.tags.forEach { add(it) } + for (mixinName in st.include) { + mixins[mixinName]?.let { mx -> + mx.tags.forEach { add(it) } + mx.refs.forEach { applyRef(it) } + } + } + st.refs.forEach { applyRef(it) } + } + val chainNames = chain.map { it.name }.toSet() + for (mx in mixins.values) { + if (mx.appliesAll || mx.appliesTo.any { it in chainNames }) { + mx.tags.forEach { add(it) } + mx.refs.forEach { if (it.name !in result) add(materialize(it)) } + } + } + return result.values.toList() + } + + /** Returns the tags declared in `trace_level`, which are set once per trace, not per span. */ + fun traceLevelTags(): List = traceLevel + + /** Includes every declaration, even from mixins whose span types are not modeled yet. */ + fun allDeclaredTags(): List = buildList { + addAll(traceLevel) + spanTypes.toSortedMap().values.forEach { addAll(it.tags) } + mixins.toSortedMap().values.forEach { addAll(it.tags) } + }.distinctBy { it.name } + + /** + * Returns mixins with `applies` targets missing from `span_types`, paired with the missing names. + * + * These targets are reported in `resolved-tags.txt` without failing generation. The mixin's + * tags are still registered, and any matching declared span types still receive them. + */ + fun unmodeledAppliesTargets(): List>> = mixins.values + .sortedBy { it.name } + .mapNotNull { mx -> + val missing = mx.appliesTo.filter { it !in spanTypes }.sorted() + if (missing.isEmpty()) null else mx.name to missing + } + + companion object { + @Suppress("UNCHECKED_CAST") + fun parse(root: Map): TagConventions { + for (section in listOf("span_types", "mixins", "trace_level")) { + require(root[section] == null || root[section] is Map<*, *>) { "$section must be a mapping" } + } + val spanTypesRaw = (root["span_types"] as? Map) ?: emptyMap() + val spanTypes = + spanTypesRaw.mapValues { (name, v) -> + require(v is Map<*, *>) { "span type '$name' must be a mapping" } + val m = v as Map + require(m["abstract"] == null || m["abstract"] is Boolean) { + "span type '$name' abstract must be a boolean" + } + require(m["extends"] == null || m["extends"] is String) { + "span type '$name' extends must be a span type name" + } + val include = m["include"] + require(include == null || (include is List<*> && include.all { it is String })) { + "span type '$name' include must be a list of mixin names" + } + SpanType( + name = name, + abstract = (m["abstract"] as? Boolean) ?: false, + extends = m["extends"] as? String, + include = (m["include"] as? List) ?: emptyList(), + tags = tagList(m["tags"]), + refs = refList(m["tags"]), + ) + } + + val mixinsRaw = (root["mixins"] as? Map) ?: emptyMap() + val mixins = + mixinsRaw.mapValues { (name, v) -> + require(v is Map<*, *>) { "mixin '$name' must be a mapping" } + val m = v as Map + val applies = m["applies"] + require(applies == null || applies == "all" || (applies is List<*> && applies.all { it is String })) { + "mixin '$name' applies must be 'all' or a list of span types" + } + Mixin( + name = name, + appliesAll = applies == "all", + appliesTo = if (applies is List<*>) (applies as List).toSet() else emptySet(), + tags = tagList(m["tags"]), + refs = refList(m["tags"]), + ) + } + + for (spanType in spanTypes.values) { + for (included in spanType.include) { + require(included in mixins) { + "span type '${spanType.name}' includes unknown mixin '$included'" + } + } + val visited = HashSet() + var current: SpanType? = spanType + while (current != null) { + val name = current.name + require(visited.add(name)) { "span type '${spanType.name}' has cyclic extends at '$name'" } + current = current.extends?.let { parent -> + requireNotNull(spanTypes[parent]) { "span type '$name' extends unknown span type '$parent'" } + } + } + } + + // Trace-level tags pass through under their Datadog name for now; their OTel mapping (resource + // attributes) is a follow-on. TODO(otel follow-on). + val traceLevelRaw = (root["trace_level"] as? Map)?.get("tags") + val traceLevel = tagList(traceLevelRaw) + require(refList(traceLevelRaw).isEmpty()) { "trace_level tags must be declarations, not refs" } + validateSingleDeclaration(spanTypes, mixins, traceLevel) + validateOtelNameScope(spanTypes, mixins, traceLevel) + return TagConventions(spanTypes, mixins, traceLevel) + } + + /** + * Rejects multiple declarations of the same `dd-name`, including identical declarations. + * Declare shared tags once on a parent or mixin and reuse them through `ref` entries, + * which may override only `required`. References must name a declared tag. + */ + private fun validateSingleDeclaration( + spanTypes: Map, + mixins: Map, + traceLevel: List, + ) { + val home = HashMap() // name -> declaring container + val declare = { container: String, t: Tag -> + val prev = home.putIfAbsent(t.name, container) + require(prev == null) { + "tag '${t.name}' is declared in both '$prev' and '$container'. Declare it once and use " + + "`{ ref: ${t.name}, required: }` elsewhere; a ref may override only `required`." + } + } + traceLevel.forEach { declare("", it) } + spanTypes.values.forEach { st -> st.tags.forEach { declare(st.name, it) } } + mixins.values.forEach { mx -> mx.tags.forEach { declare("mixin ${mx.name}", it) } } + + val refs = + spanTypes.values.flatMap { st -> st.refs.map { st.name to it } } + + mixins.values.flatMap { mx -> mx.refs.map { "mixin ${mx.name}" to it } } + for ((container, r) in refs) { + require(r.name in home) { "'$container' refs undeclared tag '${r.name}'" } + } + } + + /** + * Requires `span-kind-neutral: true` for renames on concrete span types because name + * resolution ignores span kind. Validation checks the flag and requires a configured + * rename, relying on the author's semantic check. Shared scopes do not require the flag. + */ + private fun validateOtelNameScope( + spanTypes: Map, + mixins: Map, + traceLevel: List, + ) { + val all = traceLevel + spanTypes.values.flatMap { it.tags } + mixins.values.flatMap { it.tags } + for (t in all) { + require(!t.spanKindNeutral || t.otelName != null) { + "tag '${t.name}' sets span-kind-neutral without an otel-name" + } + } + for (st in spanTypes.values.filter { !it.abstract }) { + for (t in st.tags) { + require(t.otelName == null || t.spanKindNeutral) { + "tag '${t.name}' renames to otel-name '${t.otelName}' on concrete span type '${st.name}'. " + + "Canonicalization ignores span kind, so either declare it in a shared scope (an " + + "abstract parent or a mixin) or, if '${t.otelName}' means '${t.name}' on every span " + + "kind, add `span-kind-neutral: true`." + } + } + } + } + + private val REF_KEYS = setOf("ref", "required") + + @Suppress("UNCHECKED_CAST") + private fun refList(tags: Any?): List = (tags as? List>) + ?.filter { it.containsKey("ref") } + ?.map { m -> + val extra = m.keys - REF_KEYS + require(extra.isEmpty()) { + "ref '${m["ref"]}' may override only `required`, but also sets $extra; identity " + + "(type, otel-name) comes from the tag's single declaration" + } + val name = m["ref"] + require(name is String && name.isNotBlank()) { "ref has no valid tag name: $m" } + require(m["required"] == null || m["required"] is String) { "ref '$name' required must be a string" } + Ref(name, m["required"] as? String) + } ?: emptyList() + + @Suppress("UNCHECKED_CAST") + private fun tagList(tags: Any?): List { + require(tags == null || tags is List<*>) { "tags must be a list of tag declarations" } + return (tags as? List<*>)?.mapNotNull { entry -> + require(entry is Map<*, *>) { "tag declaration must be a mapping" } + val m = entry as Map + if (m.containsKey("ref")) return@mapNotNull null + val name = parseDdName(m) + require(m["type"] == null || m["type"] is String) { "tag '$name' type must be a string" } + require(m["required"] == null || m["required"] is String) { "tag '$name' required must be a string" } + Tag( + name = name, + type = (m["type"] as? String) ?: "string", + required = (m["required"] as? String) ?: "optional", + otelName = parseOtelName(m), + spanKindNeutral = parseSpanKindNeutral(m), + ) + } ?: emptyList() + } + + /** Reads the required, nonblank string `dd-name`; invalid values fail generation. */ + private fun parseDdName(m: Map): String { + val raw = m["dd-name"] + require(raw is String && raw.isNotBlank()) { "tag declaration has no valid dd-name: $m" } + return raw + } + + private fun parseSpanKindNeutral(m: Map): Boolean { + if (!m.containsKey("span-kind-neutral")) return false + val raw = m["span-kind-neutral"] + require(raw is Boolean) { "tag '${m["dd-name"]}' has a non-boolean span-kind-neutral: '$raw'" } + return raw + } + + /** + * Reads `otel-name`: an omitted key or `none` returns null; otherwise returns a nonblank string. + * Rejects explicit null, blank strings, and non-string values so typos cannot disable a rename. + */ + private fun parseOtelName(m: Map): String? { + if (!m.containsKey("otel-name")) return null // absent => pass-through + val raw = m["otel-name"] + require(raw is String && raw.isNotBlank()) { + "tag '${m["dd-name"]}' has an invalid otel-name: '$raw'. Use a non-empty name, the literal " + + "`none`, or omit the key entirely for pass-through under the Datadog name." + } + return raw.takeUnless { it == "none" } + } + } +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt new file mode 100644 index 00000000000..5f12adee3b9 --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt @@ -0,0 +1,87 @@ +package datadog.buildlogic.tagRegistry + +/** + * Assigns tag ids from a parsed [TagConventions]. The id encoding mirrors KnownTagCodec: [63-48 + * serial][47-32 reserved][31-0 flags]. + * + *

An id is IDENTITY only: a globally unique serial plus the trace-level classification bit. It + * carries no storage-layout coordinate -- bits [47-32] are held vacant for the co-occurrence slot + * that the dense tag store assigns by graph coloring, which lands with the dense store itself. + * Nothing here needs to know how (or whether) a tag is stored. + * + *

Nor does anything here know how a tag is SET. Whether the tracer intercepts a tag on the + * set-path (routing it to a span field or a sampling directive instead of tag storage) is a + * property of TagInterceptor, not of the tag's identity, and modelling it was the source of a whole + * class of drift between this registry and the interceptor's actual switch. It arrives with the + * work that consumes it -- the id->handler dispatch table that retires TagInterceptor -- where the + * interceptor can be the authority. Re-adding a classification bit then is purely additive. + */ +class TagRegistry private constructor(val tags: List) { + data class Tag( + val name: String, + val type: String, + val required: String, + val serial: Int, + val traceLevel: Boolean, + val id: Long, + val otelName: String? = null, + ) + + companion object { + const val FIRST_SERIAL = 1 + const val LEVEL_TRACE = 1L shl 2 // low-32 carve bit 2; mirrors KnownTagCodec.LEVEL_TRACE + + /** + * Mirrors KnownTagCodec.makeTagId(serial) + traceLevel() -- must stay in sync. LEVEL_TRACE at + * bit 2, other low bits and the reserved [47-32] window zero. + */ + fun encode(serial: Int, traceLevel: Boolean): Long { + var id = serial.toLong() shl 48 + if (traceLevel) id = id or LEVEL_TRACE + return id + } + + fun build(conv: TagConventions): TagRegistry { + val traceNames = conv.traceLevelTags().map { it.name }.toSet() + + // Stable order (by name) so serials -- and therefore ids -- are a pure function of the input. + val tags = + conv.allDeclaredTags().sortedBy { it.name }.mapIndexed { i, t -> + val serial = FIRST_SERIAL + i + val traceLevel = t.name in traceNames + Tag( + t.name, + t.type, + t.required, + serial, + traceLevel, + id = encode(serial, traceLevel), + otelName = t.otelName + ) + } + + validateOtelNames(tags) + return TagRegistry(tags) + } + + /** + * An OpenTelemetry name must be unambiguous: it may not collide with any canonical tag name, nor + * be claimed by two different tags. Otherwise keyOf(otelName) would have no single right answer. + * Fail the build loudly rather than silently pick a winner. + */ + private fun validateOtelNames(tags: List) { + val canonical = tags.map { it.name }.toSet() + val owner = HashMap() + for (t in tags) { + val otel = t.otelName ?: continue + require(otel !in canonical) { + "OpenTelemetry name '$otel' (of '${t.name}') collides with canonical tag name '$otel'" + } + val prev = owner.put(otel, t.name) + require(prev == null) { + "OpenTelemetry name '$otel' is claimed by both '$prev' and '${t.name}'" + } + } + } + } +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryExtension.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryExtension.kt new file mode 100644 index 00000000000..f6d07ca586b --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryExtension.kt @@ -0,0 +1,25 @@ +package datadog.buildlogic.tagRegistry + +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.ProjectLayout +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory +import javax.inject.Inject + +/** Extension configuring the tag-registry generator inputs/outputs. */ +abstract class TagRegistryExtension @Inject constructor( + objectFactory: ObjectFactory, + layout: ProjectLayout +) { + /** + * Define the location of the tag conventions + */ + abstract val tagConventionsFile: RegularFileProperty + + /** + * Destination of the generated sources, by convention under `build/generated/tag-registry`. + */ + val destinationDirectory: DirectoryProperty = objectFactory.directoryProperty().convention( + layout.buildDirectory.dir("generated/tag-registry") + ) +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt new file mode 100644 index 00000000000..3d2885b3113 --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt @@ -0,0 +1,98 @@ +package datadog.buildlogic.tagRegistry + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import java.io.File +import java.util.Locale + +/** Emits the Java tag registry and reports in a deterministic order. */ +object TagRegistryGenerator { + /** Parses the conventions YAML and writes the full generated tree under [outDir]. */ + fun generate(tagConventionsFile: File, outDir: File) { + val mapper = ObjectMapper(YAMLFactory()) + val domain: Map = + tagConventionsFile.inputStream().use { + mapper.readValue(it, object : TypeReference>() {}) + } + + // Validate before touching the destination tree: an invalid domain model must fail loudly, + // not after the previous (valid) generated output has already been wiped out. + val conv = TagConventions.parse(domain) + val reg = TagRegistry.build(conv) + + // Remove obsolete generated files when the output changes. + outDir.deleteRecursively() + outDir.mkdirs() + // KnownTags.java goes under java/ (added as a srcDir); the .txt reports sit at the root. + val javaPkg = File(outDir, "java/datadog/trace/api").apply { mkdirs() } + + File(outDir, "resolved-tags.txt").writeText(resolvedReport(conv)) + File(outDir, "tag-assignment.txt").writeText(assignmentReport(reg)) + File(javaPkg, "KnownTags.java") + .writeText(KnownTagsEmitter.emit(reg, "datadog.trace.api", "KnownTags")) + } + + /** resolved-tags.txt — the per-type resolved sets (composition check). */ + private fun resolvedReport(conv: TagConventions) = buildString { + appendLine("# Resolved per-type tag sets (concrete span types).") + val unmodeled = conv.unmodeledAppliesTargets() + if (unmodeled.isNotEmpty()) { + appendLine( + """ + # + # LAYOUT GAP: these mixins apply to span types not modeled here, so they + # contribute to no resolved set below. Their tags ARE registered (an id is + # identity, not layout) -- they simply occupy no per-type slot yet. + """.trimIndent() + ) + for ((mixin, missing) in unmodeled) { + appendLine("# $mixin -> ${missing.joinToString(", ")}") + } + } + for (type in conv.concreteTypes()) { + val tags = conv.resolve(type) + appendLine( + """ + + $type (${tags.size} tags): + """.trimIndent() + ) + for (t in tags) appendLine(" - ${t.name}") + } + } + + /** tag-assignment.txt — serials, ids, and the OpenTelemetry name mapping (identity check). */ + private fun assignmentReport(reg: TagRegistry) = buildString { + appendLine( + """ + # Tag id assignment. tags=${reg.tags.size} + + # TAGS serial lvl id required name + """.trimIndent() + ) + for ((name, _, required, serial, traceLevel, id) in reg.tags) { + appendLine( + " %6d %s %-18s %-12s %s".format( + Locale.ROOT, + serial, + if (traceLevel) "T" else "-", + "0x%016X".format(Locale.ROOT, id), + required, + name + ) + ) + } + appendLine( + """ + + # OPENTELEMETRY NAMES. keyOf(otelName) resolves to the canonical tag's id; nameOf still + # returns the Datadog name, openTelemetryNameOf returns the name below. (No distinct id.) + """.trimIndent() + ) + val otelPairs = reg.tags.mapNotNull { t -> t.otelName?.let { it to t.name } }.sortedBy { it.first } + for ((otel, canonical) in otelPairs) { + appendLine(" %-30s -> %s".format(Locale.ROOT, otel, canonical)) + } + } +} diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPlugin.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPlugin.kt new file mode 100644 index 00000000000..c350d3c302e --- /dev/null +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPlugin.kt @@ -0,0 +1,29 @@ +package datadog.buildlogic.tagRegistry + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.SourceSetContainer +import org.gradle.kotlin.dsl.configure +import org.gradle.kotlin.dsl.create +import org.gradle.kotlin.dsl.register + +/** Generates the tag registry and adds its Java output to the main source set. */ +class TagRegistryGeneratorPlugin : Plugin { + override fun apply(project: Project) { + val ext = project.extensions.create("tagRegistry") + val generate = + project.tasks.register("generateKnownTags") { + group = "build" + description = "Generates the Java tag registry and assignment reports." + tagConventionsFile.convention(ext.tagConventionsFile) + destinationDirectory.convention(ext.destinationDirectory) + } + project.pluginManager.withPlugin("java") { + project.extensions.configure { + named("main") { + java.srcDir(generate.flatMap { it.destinationDirectory.dir("java") }) + } + } + } + } +} diff --git a/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPluginTest.kt b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPluginTest.kt new file mode 100644 index 00000000000..fa08bc5b177 --- /dev/null +++ b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorPluginTest.kt @@ -0,0 +1,278 @@ +package datadog.buildlogic.tagRegistry + +import org.assertj.core.api.Assertions.assertThat +import org.gradle.testkit.runner.GradleRunner +import org.gradle.testkit.runner.TaskOutcome.FROM_CACHE +import org.gradle.testkit.runner.TaskOutcome.SUCCESS +import org.gradle.testkit.runner.TaskOutcome.UP_TO_DATE +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import org.tabletest.junit.TableTest +import java.io.File +import java.net.URLClassLoader +import java.util.jar.JarFile + +class TagRegistryGeneratorPluginTest { + @TempDir lateinit var directory: File + + @TableTest( + """ + scenario | plugins + Java first | 'java; id("dd-trace-java.tag-registry-generator")' + tag registry first | 'id("dd-trace-java.tag-registry-generator"); java' + """ + ) + fun `Java compilation and source consumers generate the registry in either plugin order`( + plugins: String + ) { + gradleProject( + """ + plugins { $plugins } + java { withSourcesJar() } + """ + ) + writeJavaDependencies() + + val result = runner("compileJava", "sourcesJar", "javadoc").build() + + assertThat(result.task(":generateKnownTags")?.outcome).isEqualTo(SUCCESS) + assertThat(result.task(":compileJava")?.outcome).isEqualTo(SUCCESS) + assertThat(File(directory, "build/classes/java/main/datadog/trace/api/KnownTags.class")).exists() + assertGeneratedFiles() + assertThat(File(directory, "src/generated")).doesNotExist() + JarFile(File(directory, "build/libs/fixture-sources.jar")).use { jar -> + assertThat(jar.getEntry("datadog/trace/api/KnownTags.java")).isNotNull() + } + assertThat(File(directory, "build/docs/javadoc/datadog/trace/api/KnownTags.html")).exists() + } + + @Test + fun `tag names containing Java special characters compile and retain their values`() { + gradleProject( + """ + plugins { + java + id("dd-trace-java.tag-registry-generator") + } + """ + ) + directory.conventionsFile( + """ + span_types: + base: + tags: + - dd-name: 'tag"\name' + otel-name: "alias\"\\name\nline" + span-kind-neutral: true + """ + ) + writeJavaDependencies() + + val result = runner("compileJava").build() + + assertThat(result.task(":compileJava")?.outcome).isEqualTo(SUCCESS) + assertGeneratedFiles() + val classes = File(directory, "build/classes/java/main").toURI().toURL() + URLClassLoader(arrayOf(classes), null).use { loader -> + val names = loader.loadClass("datadog.trace.api.KnownTags").fields + .filter { it.type == String::class.java }.map { it.get(null) } + assertThat(names).containsExactlyInAnyOrder("tag\"\\name", "alias\"\\name\nline") + } + } + + @Test + fun `compilation reuses the configuration cache and regenerates after YAML changes`() { + gradleProject( + """ + plugins { + java + id("dd-trace-java.tag-registry-generator") + } + """ + ) + writeJavaDependencies() + assertThat(runner("compileJava").build().task(":generateKnownTags")?.outcome) + .isEqualTo(SUCCESS) + + val second = runner("compileJava").build() + assertThat(second.output).contains("Reusing configuration cache.") + assertThat(second.task(":generateKnownTags")?.outcome).isEqualTo(UP_TO_DATE) + + directory.conventionsFile( + """ + span_types: + base: + tags: [{dd-name: updated}] + """ + ) + val third = runner("compileJava").build() + assertThat(third.output).contains("Reusing configuration cache.") + assertThat(third.task(":generateKnownTags")?.outcome).isEqualTo(SUCCESS) + assertThat(third.task(":compileJava")?.outcome).isEqualTo(SUCCESS) + assertGeneratedFiles() + assertThat(generated().readText()).contains("UPDATED_NAME").doesNotContain("FOO_NAME") + } + + @Test + fun `generation can be restored from cache in a different project directory`() { + val firstProject = File(directory, "first") + val secondProject = File(directory, "second") + gradleProject(projectDir = firstProject) + gradleProject(projectDir = secondProject) + val cache = File(directory, "cache").toURI() + val settings = """ + rootProject.name = "fixture" + buildCache { local { directory = uri("$cache") } } + """ + firstProject.writeFile("settings.gradle.kts", settings) + secondProject.writeFile("settings.gradle.kts", settings) + + assertThat( + runner("generateKnownTags", "--build-cache", projectDir = firstProject) + .build().task(":generateKnownTags")?.outcome + ).isEqualTo(SUCCESS) + val result = runner("generateKnownTags", "--build-cache", projectDir = secondProject).build() + + assertThat(result.task(":generateKnownTags")?.outcome).isEqualTo(FROM_CACHE) + assertGeneratedFiles(File(firstProject, "build/generated/tag-registry")) + assertGeneratedFiles(File(secondProject, "build/generated/tag-registry")) + assertThat(generated(secondProject).readText()).isEqualTo(generated(firstProject).readText()) + } + + @Test + fun `output follows a custom build directory and clean removes it`() { + gradleProject( + """ + plugins { + id("dd-trace-java.tag-registry-generator") + java + } + layout.buildDirectory.set(layout.projectDirectory.dir("custom-build")) + """ + ) + writeJavaDependencies() + + val result = runner("compileJava").build() + + assertThat(result.task(":generateKnownTags")?.outcome).isEqualTo(SUCCESS) + val output = File(directory, "custom-build/generated/tag-registry") + assertGeneratedFiles(output) + assertThat(File(directory, "build/generated")).doesNotExist() + runner("clean").build() + assertThat(output).doesNotExist() + } + + @Test + fun `input and output can be overridden without losing the producer dependency`() { + gradleProject( + """ + plugins { + java + id("dd-trace-java.tag-registry-generator") + } + tagRegistry { + tagConventionsFile.set(layout.projectDirectory.file("custom.yaml")) + destinationDirectory.set(layout.buildDirectory.dir("custom-generated")) + } + """ + ) + File(directory, "tag-conventions.yaml").renameTo(File(directory, "custom.yaml")) + writeJavaDependencies() + + val result = runner("compileJava").build() + + assertThat(result.task(":generateKnownTags")?.outcome).isEqualTo(SUCCESS) + assertGeneratedFiles(File(directory, "build/custom-generated")) + assertThat(File(directory, "build/classes/java/main/datadog/trace/api/KnownTags.class")).exists() + } + + @Test + fun `help does not read or generate the domain model`() { + gradleProject() + File(directory, "tag-conventions.yaml").delete() + + val result = runner("help").build() + + assertThat(result.task(":generateKnownTags")).isNull() + assertThat(generated()).doesNotExist() + } + + private fun gradleProject( + build: String = """ + plugins { id("dd-trace-java.tag-registry-generator") } + """, + projectDir: File = directory + ) { + projectDir.writeFile( + "settings.gradle.kts", + """ + rootProject.name = "fixture" + """ + ) + projectDir.writeFile( + "build.gradle.kts", + build, + """ + tagRegistry { + tagConventionsFile.convention(layout.projectDirectory.file("tag-conventions.yaml")) + } + """ + ) + projectDir.conventionsFile( + """ + span_types: + base: + tags: [{dd-name: foo}, {dd-name: foo.name}] + """ + ) + } + + private fun runner(vararg arguments: String, projectDir: File = directory): GradleRunner = GradleRunner.create().withProjectDir(projectDir).withPluginClasspath() + .withArguments(*arguments, "--configuration-cache", "--stacktrace") + + private fun generated(projectDir: File = directory): File = File(projectDir, "build/generated/tag-registry/java/datadog/trace/api/KnownTags.java") + + private fun assertGeneratedFiles(output: File = File(directory, "build/generated/tag-registry")) { + assertThat(File(output, "java/datadog/trace/api/KnownTags.java")).isFile() + assertThat(File(output, "resolved-tags.txt")).isFile() + assertThat(File(output, "tag-assignment.txt")).isFile() + } + + // Only the API surface is needed here; internal-api tests exercise the real resolver at runtime. + private fun writeJavaDependencies() { + directory.writeFile( + "src/main/java/datadog/trace/api/KnownTagCodec.java", + """ + package datadog.trace.api; + public final class KnownTagCodec { + public interface Resolver { + String nameOf(long id); + String openTelemetryNameOf(long id); + long keyOf(String name); + } + public static int serialNum(long id) { return (int) (id >>> 48); } + } + """ + ) + directory.writeFile( + "src/main/java/datadog/trace/util/StringIndex.java", + """ + package datadog.trace.util; + public final class StringIndex { + public static final class Data { + public final int[] hashes; + public final String[] names; + public Data(String[] names) { this.names = names; this.hashes = new int[names.length]; } + } + public static final class EmbeddingSupport { + public static Data create(String[] names) { return new Data(names); } + public static int indexOf(int[] hashes, String[] names, String name) { + for (int i = 0; i < names.length; i++) if (names[i].equals(name)) return i; + return -1; + } + } + } + """ + ) + } +} diff --git a/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt new file mode 100644 index 00000000000..e05b4ce75a6 --- /dev/null +++ b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt @@ -0,0 +1,305 @@ +package datadog.buildlogic.tagRegistry + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatIllegalArgumentException +import org.intellij.lang.annotations.Language +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import org.tabletest.junit.TableTest +import java.io.File + +class TagRegistryGeneratorTest { + @TempDir + lateinit var directory: File + + @Test + fun `generation is deterministic and removes obsolete files`() { + val yaml = directory.conventionsFile( + """ + trace_level: + tags: [{dd-name: env}] + span_types: + base: + abstract: true + tags: [{dd-name: service, otel-name: service.name}] + http.server: + extends: base + include: [peer] + tags: [{dd-name: http.method, otel-name: http.request.method, span-kind-neutral: true}] + mixins: + common: + applies: all + tags: [{dd-name: span.kind}] + peer: + tags: [{dd-name: peer.service}] + ci_visibility: + applies: [test] + tags: [{dd-name: test.name}] + """ + ) + val output = File(directory, "generated") + TagRegistryGenerator.generate(yaml, output) + val first = contents(output) + output.writeFile("obsolete.txt", "obsolete") + + TagRegistryGenerator.generate(yaml, output) + + assertThat(contents(output)).isEqualTo(first) + assertThat(first.keys).containsExactlyInAnyOrder( + "java/datadog/trace/api/KnownTags.java", + "resolved-tags.txt", + "tag-assignment.txt" + ) + assertThat(first.getValue("resolved-tags.txt")) + .contains("http.server (4 tags):", " - service", " - peer.service", " - span.kind", "ci_visibility -> test") + assertThat(first.getValue("tag-assignment.txt")) + .contains("tags=6", "test.name", "service.name") + assertThat(first.getValue("java/datadog/trace/api/KnownTags.java")) + .contains("ENV_ID = 0x0001000000000004L", "TEST_NAME", "http.request.method") + } + + @TableTest( + """ + scenario | tag | message + missing Datadog name | '{type: string}' | no valid dd-name + empty Datadog name | "{dd-name: ''}" | no valid dd-name + nonstring Datadog name | '{dd-name: 42}' | no valid dd-name + null OpenTelemetry name | '{dd-name: foo, otel-name: null}' | invalid otel-name + empty OpenTelemetry name | "{dd-name: foo, otel-name: ''}" | invalid otel-name + nonstring type | '{dd-name: foo, type: 42}' | type must be a string + nonstring required | '{dd-name: foo, required: 42}' | required must be a string + """ + ) + fun `invalid declarations preserve previous output`(tag: String, message: String) { + val yaml = directory.conventionsFile("span_types: {base: {tags: [{dd-name: valid}]}}") + val output = File(directory, "generated") + TagRegistryGenerator.generate(yaml, output) + val previous = contents(output) + directory.conventionsFile("span_types: {base: {tags: [$tag]}}") + + assertThatIllegalArgumentException().isThrownBy { TagRegistryGenerator.generate(yaml, output) } + .withMessageContaining(message) + + assertThat(contents(output)).isEqualTo(previous) + } + + @TableTest( + """ + scenario | tags | message + alias matches canonical | '[{dd-name: foo, otel-name: bar}, {dd-name: bar}]' | collides with canonical + alias matches own name | '[{dd-name: foo, otel-name: foo}]' | collides with canonical + alias shared by tags | '[{dd-name: foo, otel-name: alias}, {dd-name: bar, otel-name: alias}]' | claimed by both + duplicate declaration | '[{dd-name: foo, otel-name: first}, {dd-name: foo, otel-name: second}]' | declared in both + """ + ) + fun `duplicate declarations and ambiguous OpenTelemetry names are rejected`(tags: String, message: String) { + val yaml = directory.conventionsFile("span_types: {base: {abstract: true, tags: $tags}}") + + assertThatIllegalArgumentException() + .isThrownBy { TagRegistryGenerator.generate(yaml, File(directory, "generated")) } + .withMessageContaining(message) + } + + @TableTest( + """ + scenario | domain | message + self inheritance | span_types: {base: {extends: base}} | cyclic extends + cyclic inheritance | span_types: {a: {extends: b}, b: {extends: a}} | cyclic extends + unknown parent | span_types: {base: {extends: missing}} | extends unknown span type 'missing' + unknown mixin | span_types: {base: {include: [missing]}} | includes unknown mixin 'missing' + null span type | span_types: {base: null} | span type 'base' must be a mapping + null mixin | mixins: {peer: null} | mixin 'peer' must be a mapping + scalar applies | mixins: {peer: {applies: test}} | applies must be 'all' or a list + numeric applies | mixins: {peer: {applies: 42}} | applies must be 'all' or a list + nonstring applies | mixins: {peer: {applies: [42]}} | applies must be 'all' or a list + nonmapping span types | span_types: [] | span_types must be a mapping + nonmapping mixins | mixins: [] | mixins must be a mapping + nonmapping trace level | trace_level: [] | trace_level must be a mapping + scalar include | span_types: {base: {include: peer}} | include must be a list of mixin names + nonstring include | span_types: {base: {include: [42]}} | include must be a list of mixin names + list parent | span_types: {base: {extends: [base]}} | extends must be a span type name + numeric parent | span_types: {base: {extends: 42}} | extends must be a span type name + string abstract | span_types: {base: {abstract: 'true'}} | abstract must be a boolean + mapping tags | span_types: {base: {tags: {dd-name: x}}} | tags must be a list of tag declarations + scalar tag | span_types: {base: {tags: [foo]}} | tag declaration must be a mapping + empty reference | span_types: {base: {tags: [{ref: ''}]}} | ref has no valid tag name + nonstring reference required | span_types: {base: {tags: [{ref: foo, required: 42}]}} | required must be a string + scalar trace tag | trace_level: {tags: [foo]} | tag declaration must be a mapping + trace reference | trace_level: {tags: [{ref: foo}]} | trace_level tags must be declarations + """ + ) + fun `invalid composition preserves previous output`(domain: String, message: String) { + val yaml = directory.conventionsFile("span_types: {base: {tags: [{dd-name: valid}]}}") + val output = File(directory, "generated") + TagRegistryGenerator.generate(yaml, output) + val previous = contents(output) + directory.conventionsFile(domain) + + assertThatIllegalArgumentException() + .isThrownBy { TagRegistryGenerator.generate(yaml, output) } + .withMessageContaining(message) + + assertThat(contents(output)).isEqualTo(previous) + } + + @ParameterizedTest + @ValueSource(strings = ["url.full", "url.path"]) + fun `duplicate declarations across groups are rejected even when they agree`(otelName: String) { + val yaml = directory.conventionsFile( + """ + span_types: + client: + tags: [{dd-name: http.url, otel-name: url.full}] + server: + tags: [{dd-name: http.url, otel-name: $otelName}] + """ + ) + + assertThatIllegalArgumentException() + .isThrownBy { TagRegistryGenerator.generate(yaml, File(directory, "generated")) } + .withMessageContaining("declared in both 'client' and 'server'") + } + + @Test + fun `ref overrides only the requirement level on the referencing type`() { + val conv = tagConventions( + """ + span_types: + base: + abstract: true + tags: [{dd-name: http.url, type: string, required: required, otel-name: url.full}] + server: + extends: base + client: + extends: base + tags: [{ref: http.url, required: conditional}] + """ + ) + + val onClient = conv.resolve("client").single { it.name == "http.url" } + val onServer = conv.resolve("server").single { it.name == "http.url" } + assertThat(onClient.required).isEqualTo("conditional") + assertThat(onClient.otelName).isEqualTo("url.full") + assertThat(onServer.required).isEqualTo("required") + assertThat(conv.allDeclaredTags().map { it.name }).containsExactly("http.url") + } + + @Test + fun `ref adds a tag declared by an unrelated span type`() { + val conv = tagConventions( + """ + span_types: + server: + tags: [{dd-name: http.url, type: string, required: required}] + client: + tags: [{ref: http.url}] + """ + ) + + assertThat(conv.resolve("client").map { it.name to it.required }) + .containsExactly("http.url" to "required") + } + + @TableTest( + """ + scenario | tags | message + reference overrides alias | '[{dd-name: http.url, type: string}, {ref: http.url, otel-name: url.full}]' | may override only `required` + undeclared reference | '[{ref: http.url}]' | refs undeclared tag 'http.url' + """ + ) + fun `invalid refs are rejected`(tags: String, message: String) { + val yaml = directory.conventionsFile("span_types: {client: {tags: $tags}}") + + assertThatIllegalArgumentException() + .isThrownBy { TagRegistryGenerator.generate(yaml, File(directory, "generated")) } + .withMessageContaining(message) + } + + @Test + fun `rename on a concrete span type requires span-kind-neutral`() { + val yaml = directory.conventionsFile( + """ + span_types: + http.server: + tags: [{dd-name: http.hostname, otel-name: server.address}] + """ + ) + + assertThatIllegalArgumentException() + .isThrownBy { TagRegistryGenerator.generate(yaml, File(directory, "generated")) } + .withMessageContaining("on concrete span type 'http.server'") + .withMessageContaining("span-kind-neutral") + } + + @Test + fun `rename on a concrete span type passes when marked span-kind-neutral`() { + val yaml = directory.conventionsFile( + """ + span_types: + db.client: + tags: [{dd-name: db.type, otel-name: db.system, span-kind-neutral: true}] + """ + ) + val output = File(directory, "generated") + + TagRegistryGenerator.generate(yaml, output) + + assertThat(contents(output).getValue("java/datadog/trace/api/KnownTags.java")) + .contains("public static final String DB_TYPE_OTEL_NAME = \"db.system\";") + } + + @Test + fun `rename in a shared scope needs no span-kind-neutral`() { + val yaml = directory.conventionsFile( + """ + span_types: + http: + abstract: true + tags: [{dd-name: http.method, otel-name: http.request.method}] + http.server: + extends: http + mixins: + peer: + tags: [{dd-name: peer.port, type: int, otel-name: server.port}] + """ + ) + val output = File(directory, "generated") + + TagRegistryGenerator.generate(yaml, output) + + assertThat(contents(output).getValue("java/datadog/trace/api/KnownTags.java")) + .contains("HTTP_METHOD_OTEL_NAME", "PEER_PORT_OTEL_NAME") + } + + @Test + fun `span-kind-neutral without an otel-name fails`() { + val yaml = directory.conventionsFile( + """ + span_types: + web: + tags: [{dd-name: http.route, span-kind-neutral: true}] + """ + ) + + assertThatIllegalArgumentException() + .isThrownBy { TagRegistryGenerator.generate(yaml, File(directory, "generated")) } + .withMessageContaining("span-kind-neutral without an otel-name") + } + + private fun tagConventions(@Language("yaml") yamlText: String) = TagConventions.parse( + ObjectMapper(YAMLFactory()).readValue( + directory.conventionsFile(yamlText), + object : TypeReference>() {} + ) + ) + + private fun contents(output: File): Map = output.walkTopDown().filter { it.isFile } + .associate { + it.relativeTo(output).invariantSeparatorsPath to it.readText() + } +} diff --git a/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TestFiles.kt b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TestFiles.kt new file mode 100644 index 00000000000..bbf8560049e --- /dev/null +++ b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TestFiles.kt @@ -0,0 +1,11 @@ +package datadog.buildlogic.tagRegistry + +import org.intellij.lang.annotations.Language +import java.io.File + +internal fun File.writeFile(path: String, vararg contents: String) = resolve(path).apply { + parentFile.mkdirs() + writeText(contents.joinToString("\n") { it.trimIndent() } + "\n") +} + +internal fun File.conventionsFile(@Language("yaml") yamlText: String) = writeFile("tag-conventions.yaml", yamlText) diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 1da7da38122..e844373db79 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -108,6 +108,7 @@ dependencies { implementation(libs.asm) implementation(libs.asm.tree) + // Keep aligned with build-logic/tag-registry, which shares these Jackson classes. implementation(platform("com.fasterxml.jackson:jackson-bom:2.17.2")) implementation("com.fasterxml.jackson.core:jackson-databind") implementation("com.fasterxml.jackson.core:jackson-annotations") diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/java/opentelemetry14/OpenTelemetry14ConventionsTest.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/java/opentelemetry14/OpenTelemetry14ConventionsTest.java index 636df381731..51d4d59452a 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/java/opentelemetry14/OpenTelemetry14ConventionsTest.java +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/java/opentelemetry14/OpenTelemetry14ConventionsTest.java @@ -40,6 +40,16 @@ public class OpenTelemetry14ConventionsTest extends AbstractOpenTelemetry14Test private static final String SPAN_KIND_INTERNAL = "internal"; private static final String OPERATION_NAME_SPECIFIC_ATTRIBUTE = "operation.name"; + // Attributes with a dd-name registered under a different otel-name in tag-conventions.yaml are + // stored (and must be asserted) under their canonical Datadog name, not the raw OTel attribute + // key. + private static final Map CANONICAL_TAG_NAMES = new HashMap<>(); + + static { + CANONICAL_TAG_NAMES.put("http.request.method", "http.method"); + CANONICAL_TAG_NAMES.put("db.system", "db.type"); + } + static Stream testSpanNameConventionsArguments() { return Stream.of( // Fallback behavior @@ -121,7 +131,8 @@ void testSpanNameConventions( attributes.forEach( (key, value) -> { if (!OPERATION_NAME_SPECIFIC_ATTRIBUTE.equals(key)) { - tagMatchers.add(tag(key, is(value))); + String tagName = CANONICAL_TAG_NAMES.getOrDefault(key, key); + tagMatchers.add(tag(tagName, is(value))); } }); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index 1de59662c8d..09698ee0c5c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -10,6 +10,7 @@ import datadog.trace.api.DDTags; import datadog.trace.api.DDTraceId; import datadog.trace.api.Functions; +import datadog.trace.api.KnownTags; import datadog.trace.api.ProcessTags; import datadog.trace.api.TagMap; import datadog.trace.api.cache.DDCache; @@ -1237,6 +1238,7 @@ Object getTag(final String key) { // maintain previously observable type of the thread name :| return threadName.toString(); case Tags.HTTP_STATUS: + case KnownTags.HTTP_STATUS_CODE_OTEL_NAME: return 0 == httpStatusCode ? null : (int) httpStatusCode; case Tags.SPAN_KIND: return getSpanKindString(); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java index 845f870b965..17963da3514 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java @@ -9,7 +9,6 @@ import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.DOUBLE_ATTRIBUTE; import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.LONG_ATTRIBUTE; import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.STRING_ATTRIBUTE; -import static datadog.trace.common.writer.RemoteMapper.HTTP_STATUS; import static datadog.trace.common.writer.ddagent.TraceMapper.ORIGIN_KEY; import static datadog.trace.common.writer.ddagent.TraceMapper.PROCESS_TAGS_KEY; import static datadog.trace.common.writer.ddagent.TraceMapper.SAMPLING_PRIORITY_KEY; @@ -26,6 +25,8 @@ import datadog.json.JsonWriter; import datadog.trace.api.Config; import datadog.trace.api.DDTags; +import datadog.trace.api.KnownTagCodec; +import datadog.trace.api.KnownTags; import datadog.trace.api.TagMap; import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; @@ -41,11 +42,27 @@ /** Provides writers for OpenTelemetry's "trace.proto" JSON encoding. */ public final class OtlpTraceJson { - private static final UTF8BytesString SERVICE_NAME = UTF8BytesString.create("service.name"); private static final UTF8BytesString RESOURCE_NAME = UTF8BytesString.create("resource.name"); private static final UTF8BytesString OPERATION_NAME = UTF8BytesString.create("operation.name"); private static final UTF8BytesString SPAN_TYPE = UTF8BytesString.create("span.type"); + /* + * Span fields and intercepted Metadata fields bypass TagMap entry projection. Resolve their + * keys through the registry here so JSON and protobuf use the same OpenTelemetry names. + */ + private static final UTF8BytesString SERVICE_NAME_KEY = otelKey(KnownTags.SERVICE_ID); + + private static final UTF8BytesString HTTP_STATUS_CODE_KEY = + otelKey(KnownTags.HTTP_STATUS_CODE_ID); + + private static final UTF8BytesString HTTP_STATUS_CODE_KEY_DD = + UTF8BytesString.create(KnownTags.HTTP_STATUS_CODE_NAME); + + /** The OpenTelemetry-namespace key for a known tag, as named by the registry. */ + private static UTF8BytesString otelKey(long tagId) { + return UTF8BytesString.create(KnownTagCodec.openTelemetryTagOf(tagId)); + } + private OtlpTraceJson() {} /** Writes one complete {@code Span} JSON object. */ @@ -88,7 +105,7 @@ public static void writeSpan( writer.name("attributes").beginArray(); if (!Config.get().getServiceName().equals(span.getServiceName())) { - writeSpanTag(writer, SERVICE_NAME, span.getServiceName()); + writeSpanTag(writer, SERVICE_NAME_KEY, span.getServiceName()); } writeSpanTag(writer, RESOURCE_NAME, span.getResourceName()); writeSpanTag(writer, OPERATION_NAME, span.getOperationName()); @@ -143,20 +160,25 @@ public static void writeSpanLink(JsonWriter writer, AgentSpanLink spanLink) { } private static void writeSpanTag(JsonWriter writer, TagMap.EntryReader tagEntry) { + // OTLP is the OpenTelemetry wire format, but the rename itself is opt-in: emit the + // OpenTelemetry-namespace name only when OTel semantics are enabled, else the entry's own + // (Datadog) name, so existing consumers keep seeing Datadog names until they opt in. + String key = + Config.get().isTraceOtelSemanticsEnabled() ? tagEntry.openTelemetryTag() : tagEntry.tag(); switch (tagEntry.type()) { case TagMap.EntryReader.BOOLEAN: - writeAttribute(writer, BOOLEAN_ATTRIBUTE, tagEntry.tag(), tagEntry.objectValue()); + writeAttribute(writer, BOOLEAN_ATTRIBUTE, key, tagEntry.objectValue()); break; case TagMap.EntryReader.INT: case TagMap.EntryReader.LONG: - writeAttribute(writer, LONG_ATTRIBUTE, tagEntry.tag(), tagEntry.objectValue()); + writeAttribute(writer, LONG_ATTRIBUTE, key, tagEntry.objectValue()); break; case TagMap.EntryReader.FLOAT: case TagMap.EntryReader.DOUBLE: - writeAttribute(writer, DOUBLE_ATTRIBUTE, tagEntry.tag(), tagEntry.objectValue()); + writeAttribute(writer, DOUBLE_ATTRIBUTE, key, tagEntry.objectValue()); break; default: - writeAttribute(writer, STRING_ATTRIBUTE, tagEntry.tag(), tagEntry.stringValue()); + writeAttribute(writer, STRING_ATTRIBUTE, key, tagEntry.stringValue()); } } @@ -211,7 +233,12 @@ public void accept(Metadata metadata) { writeSpanTag(writer, THREAD_ID, metadata.getThreadId()); writeSpanTag(writer, THREAD_NAME, metadata.getThreadName()); if (metadata.getHttpStatusCode() != UNSET_STATUS) { - writeSpanTag(writer, HTTP_STATUS, metadata.getHttpStatusCodeString()); + writeSpanTag( + writer, + Config.get().isTraceOtelSemanticsEnabled() + ? HTTP_STATUS_CODE_KEY + : HTTP_STATUS_CODE_KEY_DD, + metadata.getHttpStatusCode()); } if (metadata.getOrigin() != null) { writeSpanTag(writer, ORIGIN_KEY, metadata.getOrigin()); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java index 707225cf5a6..e7b0310bc42 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java @@ -9,7 +9,6 @@ import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.DOUBLE_ATTRIBUTE; import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.LONG_ATTRIBUTE; import static datadog.trace.bootstrap.otlp.common.OtlpAttributeVisitor.STRING_ATTRIBUTE; -import static datadog.trace.common.writer.RemoteMapper.HTTP_STATUS; import static datadog.trace.common.writer.ddagent.TraceMapper.ORIGIN_KEY; import static datadog.trace.common.writer.ddagent.TraceMapper.PROCESS_TAGS_KEY; import static datadog.trace.common.writer.ddagent.TraceMapper.SAMPLING_PRIORITY_KEY; @@ -38,6 +37,8 @@ import datadog.trace.api.Config; import datadog.trace.api.DDTags; import datadog.trace.api.DDTraceId; +import datadog.trace.api.KnownTagCodec; +import datadog.trace.api.KnownTags; import datadog.trace.api.TagMap; import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; @@ -53,11 +54,27 @@ /** Provides optimized writers for OpenTelemetry's "trace.proto" wire protocol. */ public final class OtlpTraceProto { - private static final UTF8BytesString SERVICE_NAME = UTF8BytesString.create("service.name"); private static final UTF8BytesString RESOURCE_NAME = UTF8BytesString.create("resource.name"); private static final UTF8BytesString OPERATION_NAME = UTF8BytesString.create("operation.name"); private static final UTF8BytesString SPAN_TYPE = UTF8BytesString.create("span.type"); + /* + * Span fields and intercepted Metadata fields bypass TagMap entry projection. Resolve their + * keys through the registry here so OpenTelemetry names stay consistent with entry tags. + */ + private static final UTF8BytesString SERVICE_NAME_KEY = otelKey(KnownTags.SERVICE_ID); + + private static final UTF8BytesString HTTP_STATUS_CODE_KEY = + otelKey(KnownTags.HTTP_STATUS_CODE_ID); + + private static final UTF8BytesString HTTP_STATUS_CODE_KEY_DD = + UTF8BytesString.create(KnownTags.HTTP_STATUS_CODE_NAME); + + /** The OpenTelemetry-namespace key for a known tag, as named by the registry. */ + private static UTF8BytesString otelKey(long tagId) { + return UTF8BytesString.create(KnownTagCodec.openTelemetryTagOf(tagId)); + } + private OtlpTraceProto() {} /** Records a scoped spans message after its nested span messages have been recorded. */ @@ -134,7 +151,7 @@ public static int recordSpanMessage( writeI64(buf, span.getStartTime() + PendingTrace.getDurationNano(span)); if (!Config.get().getServiceName().equals(span.getServiceName())) { - writeSpanTag(buf, SERVICE_NAME, span.getServiceName()); + writeSpanTag(buf, SERVICE_NAME_KEY, span.getServiceName()); } writeSpanTag(buf, RESOURCE_NAME, span.getResourceName()); writeSpanTag(buf, OPERATION_NAME, span.getOperationName()); @@ -208,20 +225,25 @@ public static void writeSpanId(StreamingBuffer buf, long spanId) { private static void writeSpanTag(StreamingBuffer buf, TagMap.EntryReader tagEntry) { writeTag(buf, 9, LEN_WIRE_TYPE); + // OTLP is the OpenTelemetry wire format, but the rename itself is opt-in: emit the + // OpenTelemetry-namespace name only when OTel semantics are enabled, else the entry's own + // (Datadog) name, so existing consumers keep seeing Datadog names until they opt in. + String key = + Config.get().isTraceOtelSemanticsEnabled() ? tagEntry.openTelemetryTag() : tagEntry.tag(); switch (tagEntry.type()) { case TagMap.EntryReader.BOOLEAN: - writeAttribute(buf, BOOLEAN_ATTRIBUTE, tagEntry.tag(), tagEntry.objectValue()); + writeAttribute(buf, BOOLEAN_ATTRIBUTE, key, tagEntry.objectValue()); break; case TagMap.EntryReader.INT: case TagMap.EntryReader.LONG: - writeAttribute(buf, LONG_ATTRIBUTE, tagEntry.tag(), tagEntry.objectValue()); + writeAttribute(buf, LONG_ATTRIBUTE, key, tagEntry.objectValue()); break; case TagMap.EntryReader.FLOAT: case TagMap.EntryReader.DOUBLE: - writeAttribute(buf, DOUBLE_ATTRIBUTE, tagEntry.tag(), tagEntry.objectValue()); + writeAttribute(buf, DOUBLE_ATTRIBUTE, key, tagEntry.objectValue()); break; default: - writeAttribute(buf, STRING_ATTRIBUTE, tagEntry.tag(), tagEntry.stringValue()); + writeAttribute(buf, STRING_ATTRIBUTE, key, tagEntry.stringValue()); } } @@ -288,7 +310,12 @@ public void accept(Metadata metadata) { writeSpanTag(buf, THREAD_ID, metadata.getThreadId()); writeSpanTag(buf, THREAD_NAME, metadata.getThreadName()); if (metadata.getHttpStatusCode() != UNSET_STATUS) { - writeSpanTag(buf, HTTP_STATUS, metadata.getHttpStatusCodeString()); + writeSpanTag( + buf, + Config.get().isTraceOtelSemanticsEnabled() + ? HTTP_STATUS_CODE_KEY + : HTTP_STATUS_CODE_KEY_DD, + metadata.getHttpStatusCode()); } if (metadata.getOrigin() != null) { writeSpanTag(buf, ORIGIN_KEY, metadata.getOrigin()); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/taginterceptor/TagInterceptor.java b/dd-trace-core/src/main/java/datadog/trace/core/taginterceptor/TagInterceptor.java index d81a9cc8441..89e5a01adb8 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/taginterceptor/TagInterceptor.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/taginterceptor/TagInterceptor.java @@ -4,6 +4,10 @@ import static datadog.trace.api.DDTags.MEASURED; import static datadog.trace.api.DDTags.ORIGIN_KEY; import static datadog.trace.api.DDTags.SPAN_TYPE; +import static datadog.trace.api.KnownTags.DB_STATEMENT_OTEL_NAME; +import static datadog.trace.api.KnownTags.HTTP_METHOD_OTEL_NAME; +import static datadog.trace.api.KnownTags.HTTP_STATUS_CODE_OTEL_NAME; +import static datadog.trace.api.KnownTags.HTTP_URL_OTEL_NAME; import static datadog.trace.api.sampling.PrioritySampling.USER_DROP; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.SERVLET_CONTEXT; import static datadog.trace.bootstrap.instrumentation.api.ServiceNameSources.SPLIT_BY_SERVLET_CONTEXT; @@ -103,6 +107,7 @@ public boolean needsIntercept(String tag) { switch (tag) { case DDTags.RESOURCE_NAME: case Tags.DB_STATEMENT: + case DB_STATEMENT_OTEL_NAME: case DDTags.SERVICE_NAME: case "service": case Tags.PEER_SERVICE: @@ -118,8 +123,11 @@ public boolean needsIntercept(String tag) { case ANALYTICS_SAMPLE_RATE: case Tags.ERROR: case HTTP_STATUS: + case HTTP_STATUS_CODE_OTEL_NAME: case HTTP_METHOD: + case HTTP_METHOD_OTEL_NAME: case HTTP_URL: + case HTTP_URL_OTEL_NAME: case ORIGIN_KEY: case MEASURED: case Tags.SPAN_KIND: @@ -135,6 +143,7 @@ public boolean interceptTag(DDSpanContext span, String tag, Object value) { case DDTags.RESOURCE_NAME: return interceptResourceName(span, value); case Tags.DB_STATEMENT: + case DB_STATEMENT_OTEL_NAME: return interceptDbStatement(span, value); case DDTags.SERVICE_NAME: case "service": @@ -184,10 +193,13 @@ public boolean interceptTag(DDSpanContext span, String tag, Object value) { case Tags.ERROR: return interceptError(span, value); case HTTP_STATUS: + case HTTP_STATUS_CODE_OTEL_NAME: // not set internally but may come from manual instrumentation return interceptHttpStatusCode(span, value); case HTTP_METHOD: + case HTTP_METHOD_OTEL_NAME: case HTTP_URL: + case HTTP_URL_OTEL_NAME: return interceptUrlResourceAsNameRule(span, tag, value); case ORIGIN_KEY: return interceptOrigin(span, value); @@ -205,13 +217,17 @@ public boolean interceptTag(DDSpanContext span, String tag, Object value) { private boolean interceptUrlResourceAsNameRule(DDSpanContext span, String tag, Object value) { if (shouldSetUrlResourceAsName) { - if (HTTP_METHOD.equals(tag)) { + if (HTTP_METHOD.equals(tag) || HTTP_METHOD_OTEL_NAME.equals(tag)) { final Object url = span.unsafeGetTag(HTTP_URL); if (url != null) { setResourceFromUrl(span, value.toString(), url); } - } else if (HTTP_URL.equals(tag)) { - final Object method = span.unsafeGetTag(HTTP_METHOD); + } else if (HTTP_URL.equals(tag) || HTTP_URL_OTEL_NAME.equals(tag)) { + // the method may have been set under either spelling -- see HTTP_METHOD_OTEL_NAME. + Object method = span.unsafeGetTag(HTTP_METHOD); + if (method == null) { + method = span.unsafeGetTag(HTTP_METHOD_OTEL_NAME); + } setResourceFromUrl(span, method != null ? method.toString() : null, value); } } diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollectorTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollectorTest.java index 3f836227a39..a0456e9d20e 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollectorTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollectorTest.java @@ -13,9 +13,11 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import datadog.json.JsonMapper; +import datadog.trace.api.Config; import datadog.trace.api.DDTraceId; import datadog.trace.api.TracePropagationStyle; import datadog.trace.api.sampling.PrioritySampling; @@ -38,6 +40,8 @@ import java.util.Map; import java.util.Set; import org.junit.jupiter.api.Test; +import org.mockito.AdditionalAnswers; +import org.mockito.MockedStatic; /** * Tests for {@link OtlpTraceJsonCollector}, parsing the produced JSON back with {@link JsonMapper} @@ -77,6 +81,63 @@ void singleSpanIsEncodedWithHexIdsAndCamelCaseKeys() throws IOException { assertTrue(attrKeys.contains("operation.name")); } + @Test + void tagsAreEmittedUnderTheirOpenTelemetryNameWhenOtelSemanticsEnabled() throws IOException { + // The JSON encoder is a second exporter of the same spans, so it must apply the registry's + // OpenTelemetry naming exactly as the protobuf one does: which transport protocol is configured + // must not change the attribute names a backend receives. + Set attrKeys = collectTagsAttributeKeys(true); + + assertTrue( + attrKeys.contains("http.request.method"), + "renamed tag must use its OpenTelemetry name; got " + attrKeys); + assertFalse( + attrKeys.contains("http.method"), + "renamed tag must not also appear under its Datadog name; got " + attrKeys); + assertTrue( + attrKeys.contains("custom.unregistered"), + "a tag the registry does not name passes through unchanged; got " + attrKeys); + } + + @Test + void tagsAreEmittedUnderTheirDatadogNameWhenOtelSemanticsDisabled() throws IOException { + // The OpenTelemetry rename is opt-in: with the flag off, existing consumers must keep seeing + // Datadog names, not the OpenTelemetry ones. + Set attrKeys = collectTagsAttributeKeys(false); + + assertTrue( + attrKeys.contains("http.method"), + "tag must use its Datadog name when OTel semantics are disabled; got " + attrKeys); + assertFalse( + attrKeys.contains("http.request.method"), + "tag must not appear under its OpenTelemetry name when OTel semantics are disabled; got " + + attrKeys); + assertTrue( + attrKeys.contains("custom.unregistered"), + "a tag the registry does not name passes through unchanged; got " + attrKeys); + } + + private static Set collectTagsAttributeKeys(boolean otelSemanticsEnabled) + throws IOException { + Config realConfig = Config.get(); + try (MockedStatic configMock = mockStatic(Config.class)) { + Config config = mock(Config.class, AdditionalAnswers.delegatesTo(realConfig)); + when(config.isTraceOtelSemanticsEnabled()).thenReturn(otelSemanticsEnabled); + configMock.when(Config::get).thenReturn(config); + + AgentSpan agentSpan = TRACER.startSpan("test", "op.tagged"); + agentSpan.setResourceName("GET /api"); + agentSpan.setTag("http.method", "GET"); + agentSpan.setTag("custom.unregistered", "value"); + agentSpan.setSamplingPriority(PrioritySampling.USER_KEEP, SamplingMechanism.DEFAULT); + agentSpan.finish(); + + OtlpTraceJsonCollector collector = new OtlpTraceJsonCollector(); + collector.addTrace(asList((CoreSpan) agentSpan)); + return attributeKeys(onlySpan(collector.collectTraces())); + } + } + @Test void spanKindIsEncodedAsInteger() throws IOException { DDSpan span = startAndFinish("op.server", "GET /api", SPAN_KIND_SERVER); diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceProtoTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceProtoTest.java index e326a6958b8..1402c16c7b3 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceProtoTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpTraceProtoTest.java @@ -20,12 +20,15 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import com.google.protobuf.CodedInputStream; import com.google.protobuf.WireFormat; +import datadog.trace.api.Config; import datadog.trace.api.DD128bTraceId; import datadog.trace.api.DDTraceId; +import datadog.trace.api.KnownTagCodec; import datadog.trace.api.TracePropagationStyle; import datadog.trace.api.sampling.PrioritySampling; import datadog.trace.api.sampling.SamplingMechanism; @@ -51,6 +54,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.AdditionalAnswers; +import org.mockito.MockedStatic; /** * Tests for {@link OtlpTraceProto} via {@link OtlpTraceProtoCollector#collectTraces}. @@ -457,11 +462,43 @@ static Stream cases() { kindSpan("third.span", SPAN_KIND_SERVER)))); } + /** + * Cartesian product of {@link #cases()} with both settings of the OTel-semantics flag, so every + * case is verified both with the OpenTelemetry-namespace renames applied and with them off (the + * default): the rename is opt-in, so both states must serialize correctly. + */ + @SuppressWarnings("unchecked") + static Stream casesWithOtelSemantics() { + return cases() + .flatMap( + args -> { + Object[] raw = args.get(); + String caseName = (String) raw[0]; + List specs = (List) raw[1]; + return Stream.of( + Arguments.of(caseName + " [otel semantics enabled]", specs, true), + Arguments.of(caseName + " [otel semantics disabled]", specs, false)); + }); + } + // ── parameterized test ──────────────────────────────────────────────────── @ParameterizedTest(name = "{0}") - @MethodSource("cases") - void testCollectTraces(String caseName, List specs) throws IOException { + @MethodSource("casesWithOtelSemantics") + void testCollectTraces(String caseName, List specs, boolean otelSemanticsEnabled) + throws IOException { + Config realConfig = Config.get(); + try (MockedStatic configMock = mockStatic(Config.class)) { + Config config = mock(Config.class, AdditionalAnswers.delegatesTo(realConfig)); + when(config.isTraceOtelSemanticsEnabled()).thenReturn(otelSemanticsEnabled); + configMock.when(Config::get).thenReturn(config); + + testCollectTracesImpl(caseName, specs, otelSemanticsEnabled); + } + } + + private void testCollectTracesImpl( + String caseName, List specs, boolean otelSemanticsEnabled) throws IOException { List spans = buildSpans(specs); OtlpTraceProtoCollector collector = new OtlpTraceProtoCollector(); @@ -527,7 +564,11 @@ void testCollectTraces(String caseName, List specs) throws IOException // ── verify each span ───────────────────────────────────────────────── for (int i = 0; i < spans.size(); i++) { verifySpan( - CodedInputStream.newInstance(spanBlobs.get(i)), spans.get(i), specs.get(i), caseName); + CodedInputStream.newInstance(spanBlobs.get(i)), + spans.get(i), + specs.get(i), + caseName, + otelSemanticsEnabled); } } @@ -999,7 +1040,11 @@ private static void verifyDefaultScope(CodedInputStream scope) throws IOExceptio * */ private static void verifySpan( - CodedInputStream spanData, DDSpan originalSpan, SpanSpec spec, String caseName) + CodedInputStream spanData, + DDSpan originalSpan, + SpanSpec spec, + String caseName, + boolean otelSemanticsEnabled) throws IOException { byte[] parsedTraceId = null; byte[] parsedSpanId = null; @@ -1167,11 +1212,67 @@ private static void verifySpan( "attributes must include 'service.name' when service is overridden [" + caseName + "]"); } - // extra user tags must appear as attributes + // Assert the selected namespace spelling and reject the other. HTTP status is checked + // separately because interception moves it out of the TagMap before serialization. for (String key : spec.extraTags.keySet()) { + if ("http.status_code".equals(key)) { + // Not a tag-map entry by the time it is serialized: the set path intercepts it into + // Metadata.httpStatusCode, so it never reaches the per-entry projection and instead is + // resolved through the fixed HTTP_STATUS_CODE_KEY constant (see the intercepted-status + // assertion below). + String expectedStatusKey = + otelSemanticsEnabled ? "http.response.status_code" : "http.status_code"; + String otherStatusKey = + otelSemanticsEnabled ? "http.status_code" : "http.response.status_code"; + assertTrue( + attrKeys.contains(expectedStatusKey), + "intercepted status must be emitted as '" + + expectedStatusKey + + "' [" + + caseName + + "]; got " + + attrKeys); + assertFalse( + attrKeys.contains(otherStatusKey), + "intercepted status must not also appear as '" + + otherStatusKey + + "' [" + + caseName + + "]"); + continue; + } + long id = KnownTagCodec.keyOf(key); + String otelName = id != 0L ? KnownTagCodec.openTelemetryNameOf(id) : null; + String expected = otelSemanticsEnabled && otelName != null ? otelName : key; assertTrue( - attrKeys.contains(key), - "attributes must include extra tag '" + key + "' [" + caseName + "]"); + attrKeys.contains(expected), + "attributes must include extra tag '" + + key + + "' as '" + + expected + + "' [" + + caseName + + "]; got " + + attrKeys); + if (otelSemanticsEnabled && otelName != null) { + assertFalse( + attrKeys.contains(key), + "renamed tag '" + + key + + "' must not also appear under its Datadog name [" + + caseName + + "]"); + } else if (otelName != null) { + assertFalse( + attrKeys.contains(otelName), + "tag '" + + key + + "' must not appear under its OpenTelemetry name '" + + otelName + + "' when OTel semantics are disabled [" + + caseName + + "]"); + } } if (spec.measured) { @@ -1180,11 +1281,26 @@ private static void verifySpan( "attributes must include '_dd.measured' for measured spans [" + caseName + "]"); } if (spec.httpStatusCode != 0) { + // Intercepted into Metadata.httpStatusCode rather than left in the tag map, so its name comes + // from a key constant in OtlpTraceProto (HTTP_STATUS_CODE_KEY / HTTP_STATUS_CODE_KEY_DD) and + // not from the per-entry projection. Emitted as an int attribute, matching the + // semantic-conventions type, since Metadata now carries the status as an int; under its + // OpenTelemetry name only when OTel semantics are enabled, else its Datadog name. + String expectedStatusKey = + otelSemanticsEnabled ? "http.response.status_code" : "http.status_code"; + String otherStatusKey = + otelSemanticsEnabled ? "http.status_code" : "http.response.status_code"; assertTrue( - attrKeys.contains("http.status_code"), - "attributes must include 'http.status_code' when set via setHttpStatusCode [" + attrKeys.contains(expectedStatusKey), + "attributes must include '" + + expectedStatusKey + + "' when set via setHttpStatusCode [" + caseName - + "]"); + + "]; got " + + attrKeys); + assertFalse( + attrKeys.contains(otherStatusKey), + "status code must not also be emitted as '" + otherStatusKey + "' [" + caseName + "]"); } if (spec.origin != null) { assertTrue( diff --git a/dd-trace-core/src/test/java/datadog/trace/core/taginterceptor/TagInterceptorTest.java b/dd-trace-core/src/test/java/datadog/trace/core/taginterceptor/TagInterceptorTest.java index ae85244a729..91f3da0910f 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/taginterceptor/TagInterceptorTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/taginterceptor/TagInterceptorTest.java @@ -707,6 +707,90 @@ static Stream urlAsResourceNameRuleSetsTheResourceNameArguments() { ); } + @TableTest({ + "scenario | methodTag ", + "datadog spelling | 'Tags.HTTP_METHOD' ", + "otel spelling | 'http.request.method'" + }) + void urlAsResourceNameRuleAppliesRegardlessOfMethodSpellingWhenUrlSetFirst( + @ConvertWith(TagsConverter.class) String methodTag) { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + AgentSpan span = tracer.buildSpan("datadog", "fakeOperation").start(); + try { + span.setTag(HTTP_URL, "/with-method"); + span.setTag(methodTag, "Post"); + assertEquals("POST /with-method", span.getResourceName().toString()); + } finally { + span.finish(); + } + } + + @TableTest({ + "scenario | methodTag ", + "datadog spelling | 'Tags.HTTP_METHOD' ", + "otel spelling | 'http.request.method'" + }) + void urlAsResourceNameRuleAppliesRegardlessOfMethodSpellingWhenMethodSetFirst( + @ConvertWith(TagsConverter.class) String methodTag) { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + AgentSpan span = tracer.buildSpan("datadog", "fakeOperation").start(); + try { + span.setTag(methodTag, "Post"); + span.setTag(HTTP_URL, "/with-method"); + assertEquals("POST /with-method", span.getResourceName().toString()); + } finally { + span.finish(); + } + } + + @TableTest({ + "scenario | statementTag | beforeStart", + "datadog spelling, builder | 'db.statement' | true ", + "datadog spelling, setTag | 'db.statement' | false ", + "otel spelling, builder | 'db.query.text' | true ", + "otel spelling, setTag | 'db.query.text' | false " + }) + void dbStatementIsConsumedIntoResourceRegardlessOfSpellingOrTiming( + String statementTag, boolean beforeStart) { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + AgentSpan span = + beforeStart + ? tracer.buildSpan("datadog", "fakeOperation").withTag(statementTag, "select 1").start() + : tracer.buildSpan("datadog", "fakeOperation").start(); + try { + if (!beforeStart) { + span.setTag(statementTag, "select 1"); + } + assertEquals("select 1", span.getResourceName().toString()); + assertNull(span.getTag(Tags.DB_STATEMENT)); + assertNull(span.getTag("db.query.text")); + } finally { + span.finish(); + } + } + + @TableTest({ + "scenario | urlTag ", + "datadog spelling | 'Tags.HTTP_URL'", + "otel spelling | 'url.full' " + }) + void urlAsResourceNameRuleAppliesRegardlessOfUrlSpelling( + @ConvertWith(TagsConverter.class) String urlTag) { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + AgentSpan span = tracer.buildSpan("datadog", "fakeOperation").start(); + try { + span.setTag(HTTP_METHOD, "Post"); + span.setTag(urlTag, "/with-method"); + assertEquals("POST /with-method", span.getResourceName().toString()); + } finally { + span.finish(); + } + } + @Test void whenUserSetsPeerServiceTheSourceShouldBePeerService() { CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); @@ -720,6 +804,51 @@ void whenUserSetsPeerServiceTheSourceShouldBePeerService() { } } + @TableTest({ + "scenario | tag ", + "datadog spelling | 'Tags.HTTP_STATUS' ", + "otel spelling | 'http.response.status_code'" + }) + void httpStatusCodeIsInterceptedRegardlessOfSpelling( + @ConvertWith(TagsConverter.class) String tag) { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + DDSpan span = (DDSpan) tracer.buildSpan("datadog", "fakeOperation").start(); + try { + span.setTag(tag, 200); + assertEquals((short) 200, span.getHttpStatusCode()); + // consumed into Metadata's httpStatusCode field, not left behind as a generic tag under + // either spelling -- otherwise the OTLP writers would emit it a second time. getTag(...) + // is not a suitable check here: it synthesizes the Datadog spelling's value straight from + // httpStatusCode regardless of generic storage, so go straight to the backing TagMap. + assertNull(span.unsafeGetTag(HTTP_STATUS)); + assertNull(span.unsafeGetTag("http.response.status_code")); + } finally { + span.finish(); + } + } + + @TableTest({ + "scenario | tag ", + "datadog spelling | 'Tags.HTTP_STATUS' ", + "otel spelling | 'http.response.status_code'" + }) + void httpStatusCodeIsReadableRegardlessOfSpelling(@ConvertWith(TagsConverter.class) String tag) { + CoreTracer tracer = tracerBuilder().writer(new ListWriter()).build(); + + DDSpan span = (DDSpan) tracer.buildSpan("datadog", "fakeOperation").start(); + try { + span.setTag(tag, 200); + // regardless of which spelling was used to set it, both spellings must read it back -- + // TagsMatcher (and thus trace-sampling rules) reads tags through getTag(), so a spelling + // the getter doesn't recognize would look unset even though it was intercepted. + assertEquals(200, span.getTag(Tags.HTTP_STATUS)); + assertEquals(200, span.getTag("http.response.status_code")); + } finally { + span.finish(); + } + } + @Test void whenInterceptServiceNameExtraServiceProviderIsCalled() { ServiceNameCollector origServiceNameCollector = ServiceNameCollector.get(); diff --git a/gradle/java_no_deps.gradle b/gradle/java_no_deps.gradle index d2154d39dad..b723dd5e73c 100644 --- a/gradle/java_no_deps.gradle +++ b/gradle/java_no_deps.gradle @@ -226,7 +226,7 @@ tasks.named("javadoc", Javadoc) { javadocTool = javaToolchains.javadocToolFor { languageVersion = JavaLanguageVersion.of(8) } - source = sourceSets.main.java.srcDirs + source = sourceSets.main.java classpath = configurations.compileClasspath options { diff --git a/internal-api/build.gradle.kts b/internal-api/build.gradle.kts index 7ae9ee327f4..a9d82cebc72 100644 --- a/internal-api/build.gradle.kts +++ b/internal-api/build.gradle.kts @@ -5,6 +5,7 @@ plugins { `java-library` id("dd-trace-java.module.internal-api") id("dd-trace-java.jmh-conventions") + id("dd-trace-java.tag-registry-generator") } java { @@ -26,6 +27,9 @@ extra["minimumBranchCoverage"] = 0.7 extra["minimumInstructionCoverage"] = 0.8 extra["excludedClassesCoverage"] = listOf( + // Generated by the tag-registry code generator. + "datadog.trace.api.KnownTags", + "datadog.trace.api.KnownTags.*", "datadog.trace.api.ClassloaderConfigurationOverrides", "datadog.trace.api.ClassloaderConfigurationOverrides.Lazy", // Interface @@ -260,6 +264,10 @@ extra["excludedClassesBranchCoverage"] = listOf( extra["excludedClassesInstructionCoverage"] = listOf("datadog.trace.util.stacktrace.StackWalkerFactory") +tagRegistry { + tagConventionsFile.set(rootProject.layout.projectDirectory.file("tag-conventions.yaml")) +} + dependencies { // references TraceScope and Continuation from public api api(project(":dd-trace-api")) diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java new file mode 100644 index 00000000000..d84e1e6a537 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -0,0 +1,140 @@ +package datadog.trace.api; + +/** + * Resolves generated tag IDs and names. {@link KnownTags} owns the lookup tables; this class owns + * the ID layout and namespace naming policy. + * + *

Datadog and OpenTelemetry names can resolve to the same ID. {@link #canonicalTagName(String)} + * provides the Datadog key used by {@link TagMap}; the namespace readers choose output names. The + * ID does not determine whether a tag is intercepted or where it is stored. + */ +public final class KnownTagCodec { + /* + * tagId bit layout: [63-48 serialNum (16 bits)] [47-32 reserved, zero] [31-0 flags]. serialNum is + * globally unique per known tag and is the whole of the tag's identity — nameOf/ + * openTelemetryNameOf switch on it, and the generator emits each id as a literal. Bits [47-32] + * are RESERVED and always zero here: they are the window the dense tag store uses for its + * co-occurrence slot coordinate, which arrives with that store. Of the low 32 flag bits, bit 2 is + * the trace/span LEVEL bit (set ⟹ trace-level); bits 1-0 are reserved. Unknown (string-only) + * custom tags are NOT known ids — {@code keyOf} returns 0 for them. + * + *

An id says what a tag IS, not how it is SET. Whether the tracer intercepts a tag on the + * set-path — routing it to a span field or a sampling directive instead of tag storage — belongs + * to TagInterceptor, whose {@code needsIntercept} switch is the authority; mirroring it here as a + * classification bit and a serial-range tier only created drift between the two. That + * classification returns with the work that consumes it (the id→handler dispatch table that + * retires TagInterceptor), and re-adding a bit then is purely additive. + * + *

There is deliberately NO OpenTelemetry-applicability flag: an absent otel-name means + * pass-through (the tag is emitted under its Datadog name), so today every known tag has an + * OpenTelemetry name and such a flag would be constant. It returns once a Datadog-only tag exists. + */ + public static int serialNum(long tagId) { + return (int) (tagId >>> 48); + } + + /** + * Trace/span LEVEL bit (low-32 carve, bit 2). Set marks a trace-level tag (lives on the + * TraceSegment's own TagMap); clear marks a span-level tag. Declared in the conventions as the + * {@code trace_level} tier, so it is part of the tag's identity rather than of any storage + * scheme. + */ + public static final long LEVEL_TRACE = 1L << 2; + + /** True if the tagId names a trace-level tag. */ + public static boolean isTraceLevel(long tagId) { + return (tagId & LEVEL_TRACE) != 0L; + } + + /** Returns the tagId with the {@link #LEVEL_TRACE} flag set. */ + public static long traceLevel(long tagId) { + return tagId | LEVEL_TRACE; + } + + /** + * Builds a tagId from its {@code serialNum} (globally unique per known tag). The reserved [47-32] + * window and the low 32 bits are zero, so the id is fully determined by the serial — the + * generator emits it as a literal. Inverse of {@link #serialNum}. Intended for the code generator + * and tests. + */ + public static long makeTagId(int serialNum) { + return (long) serialNum << 48; + } + + public interface Resolver { + /** The tag's Datadog-namespace (canonical) name. */ + String nameOf(long tagId); + + /** The tag's OpenTelemetry-namespace name, or {@code null} when it declares none. */ + String openTelemetryNameOf(long tagId); + + /** The id for {@code name} in ANY namespace (many→one), or 0 when it is not a known tag. */ + long keyOf(String name); + } + + /** + * Loads the generated resolver on the first name lookup. Reading {@code RESOLVER} initializes + * {@link KnownTags}, so callers cannot observe an unregistered or partially initialized registry. + * The {@code static final} receiver can also help the JIT inline resolver calls. + */ + private static final class Installed { + static final Resolver RESOLVER = KnownTags.RESOLVER; + } + + /** The tag's canonical (Datadog-namespace) name, or {@code null} when the id is not known. */ + public static String nameOf(long tagId) { + return Installed.RESOLVER.nameOf(tagId); + } + + /** The tag's Datadog-namespace (canonical) name — the same value as {@link #nameOf}. */ + public static String datadogNameOf(long tagId) { + return nameOf(tagId); + } + + /** + * The tag's declared OpenTelemetry RENAME, or {@code null} when it declares none. Raw registry + * data — it does not apply the pass-through default, so most callers want {@link + * #openTelemetryTagOf} instead. + */ + public static String openTelemetryNameOf(long tagId) { + return Installed.RESOLVER.openTelemetryNameOf(tagId); + } + + /** + * The name {@code tagId} is emitted under in the OpenTelemetry namespace: its declared rename + * when it has one, otherwise its Datadog name — pass-through, the default. {@code null} for an + * unknown id, which has no registry name at all; a custom tag falls back to its own key, and only + * the caller holding that key can do so. + * + *

This is the one place the pass-through policy lives, so no serializer re-decides it. Pair it + * with {@link #datadogNameOf} for the same tag under the Datadog namespace; outbound naming is + * per-namespace, never normalized to one of them. + */ + public static String openTelemetryTagOf(long tagId) { + Resolver resolver = Installed.RESOLVER; + String otelName = resolver.openTelemetryNameOf(tagId); + return otelName != null ? otelName : resolver.nameOf(tagId); + } + + /** The id for {@code name} in any namespace, or 0 when it is not a known tag. */ + public static long keyOf(String name) { + return Installed.RESOLVER.keyOf(name); + } + + /** + * The Datadog-namespace name to store {@code name} under: {@code name} itself when it is not a + * known tag, otherwise the canonical Datadog name for whichever id it resolves to. A Datadog name + * maps to itself (no-op); an OpenTelemetry rename maps to the Datadog name it is a rename of. + * + *

{@link TagMap} calls this at entry construction so that setting a known tag under its + * Datadog name and under its OpenTelemetry rename store to the same {@code Entry} rather than two + * separate ones -- the two names denote one tag, and only serialization ({@link #datadogNameOf}, + * {@link #openTelemetryTagOf}) should see them as different. + */ + public static String canonicalTagName(String name) { + long id = keyOf(name); + return id != 0 ? nameOf(id) : name; + } + + private KnownTagCodec() {} +} diff --git a/internal-api/src/main/java/datadog/trace/api/TagMap.java b/internal-api/src/main/java/datadog/trace/api/TagMap.java index 2f2423d5239..52d2ef32a2b 100644 --- a/internal-api/src/main/java/datadog/trace/api/TagMap.java +++ b/internal-api/src/main/java/datadog/trace/api/TagMap.java @@ -53,6 +53,10 @@ public final class TagMap implements Map, Iterable. public static final TagMap EMPTY = new TagMap(new Object[1], 0); + // Sentinel for an ID not yet resolved by the reused EntryReadingHelper. Zero denotes an unknown + // tag, so it cannot represent "not computed". + static final long TAG_ID_NOT_COMPUTED = Long.MIN_VALUE; + /** Creates a new mutable TagMap that contains the contents of map */ public static final TagMap fromMap(@Nonnull Map map) { TagMap tagMap = TagMap.create(map.size()); @@ -141,7 +145,9 @@ public final boolean matches(String tag) { public static final class EntryRemoval extends EntryChange { EntryRemoval(String tag) { - super(tag); + // Canonicalize so a removal recorded under an OpenTelemetry rename matches an Entry recorded + // (via Entry's own constructor) under its Datadog name -- see Ledger#contains/#matches. + super(KnownTagCodec.canonicalTagName(tag)); } @Override @@ -171,6 +177,24 @@ public interface EntryReader { String tag(); + /** Returns the known ID for {@link #tag()}, or {@code 0L} for an unknown tag. */ + long tagId(); + + /** + * This entry's tag name in the OpenTelemetry namespace: the rename the registry declares for + * it, else its Datadog name (pass-through, the default), else — for a custom tag, which the + * registry does not name at all — {@link #tag()} itself. + * + *

Never null, which is the point of asking the reader rather than the codec. {@link + * KnownTagCodec#openTelemetryTagOf} owns the naming policy but returns null for an unknown id, + * because only the holder of the entry knows the key to fall back to. This completes that one + * step and nothing more, so the policy still lives in exactly one place. + */ + default String openTelemetryTag() { + String otelTag = KnownTagCodec.openTelemetryTagOf(tagId()); + return otelTag != null ? otelTag : tag(); + } + byte type(); boolean is(byte type); @@ -342,7 +366,12 @@ static Entry newDoubleEntry(String tag, Double box) { volatile String strCache = null; private Entry(String tag, byte type, long prim, Object obj) { - super(tag); + /* + * Canonicalize known names at the single Entry construction point, so Datadog and + * OpenTelemetry spellings use the same TagMap key. This adds a StringIndex lookup to every + * new entry, including on the application thread. + */ + super(KnownTagCodec.canonicalTagName(tag)); this.lazyTagHash = 0; // lazily computed this.rawType = type; @@ -361,6 +390,15 @@ int hash() { return hash; } + @Override + public long tagId() { + /* + * Resolve on demand. Only OTLP serialization currently needs the ID, so caching it here + * would add a field to every Entry to save a lookup on exported tags. + */ + return KnownTagCodec.keyOf(this.tag); + } + @Override public Entry entry() { return this; @@ -914,12 +952,17 @@ public Ledger smartRemove(String tag) { } private boolean contains(String tag) { + // Entries and removals are both recorded under their canonical Datadog name (see newAnyEntry + // et al. and EntryRemoval's own constructor); canonicalize the query the same way or it + // would miss. + String canonicalTag = KnownTagCodec.canonicalTagName(tag); + EntryChange[] thisChanges = this.entryChanges; // min is to clamp, so bounds check elimination optimization works int lenClamp = Math.min(this.nextPos, thisChanges.length); for (int i = 0; i < lenClamp; ++i) { - if (thisChanges[i].matches(tag)) return true; + if (thisChanges[i].matches(canonicalTag)) return true; } return false; } @@ -928,13 +971,14 @@ private boolean contains(String tag) { * Just for testing */ Entry findLastEntry(String tag) { + String canonicalTag = KnownTagCodec.canonicalTagName(tag); EntryChange[] thisChanges = this.entryChanges; // min is to clamp, so ArrayBoundsCheckElimination optimization works int clampLen = Math.min(this.nextPos, thisChanges.length) - 1; for (int i = clampLen; i >= 0; --i) { EntryChange thisChange = thisChanges[i]; - if (!thisChange.isRemoval() && thisChange.matches(tag)) return (Entry) thisChange; + if (!thisChange.isRemoval() && thisChange.matches(canonicalTag)) return (Entry) thisChange; } return null; } @@ -1254,7 +1298,12 @@ public Set> entrySet() { } public Entry getEntry(String tag) { - Entry local = this.getLocalEntry(tag); + // Entries are stored under their canonical Datadog name (see Entry's constructor); a lookup by + // an OpenTelemetry rename must canonicalize the same way, or it would hash to the wrong bucket + // and silently miss the entry stored under the Datadog name. + String canonicalTag = KnownTagCodec.canonicalTagName(tag); + + Entry local = this.getLocalEntry(canonicalTag); if (local != null) { // Local entry shadows the parent (local-wins) — unchanged hot path. return local; @@ -1266,10 +1315,10 @@ public Entry getEntry(String tag) { if (parent == null) { return null; } - if (this.removedFromParent != null && this.removedFromParent.contains(tag)) { + if (this.removedFromParent != null && this.removedFromParent.contains(canonicalTag)) { return null; // tombstoned: removed locally, do not read through } - return parent.getEntry(tag); + return parent.getEntry(canonicalTag); } /** Looks up an entry in this map's own buckets only — no read-through to the parent. */ @@ -1734,7 +1783,11 @@ public boolean remove(String tag) { public Entry getAndRemove(String tag) { this.checkWriteAccess(); - Entry localRemoved = this.removeLocal(tag); + // See getEntry: entries are stored under their canonical Datadog name, so a removal by an + // OpenTelemetry rename must canonicalize first to find (and tombstone) the right entry. + String canonicalTag = KnownTagCodec.canonicalTagName(tag); + + Entry localRemoved = this.removeLocal(canonicalTag); TagMap parent = this.parent; if (parent != null) { @@ -1743,16 +1796,16 @@ public Entry getAndRemove(String tag) { // local entry if there was one, otherwise the parent's (which we now hide). Single-parent in // phase 1; rare path (only when removing a parent-exposed key). boolean alreadyTombstoned = - this.removedFromParent != null && this.removedFromParent.contains(tag); + this.removedFromParent != null && this.removedFromParent.contains(canonicalTag); if (!alreadyTombstoned) { - Entry parentEntry = parent.getEntry(tag); + Entry parentEntry = parent.getEntry(canonicalTag); if (parentEntry != null) { if (this.removedFromParent == null) { // Small initial capacity: this set is rare and almost always holds only a handful of // tombstoned keys, so the default 16-bucket HashSet table would be wasteful. this.removedFromParent = new HashSet<>(4); } - this.removedFromParent.add(tag); + this.removedFromParent.add(canonicalTag); return localRemoved != null ? localRemoved : parentEntry; } } @@ -2855,17 +2908,20 @@ final class EntryReadingHelper implements TagMap.EntryReader { private Map.Entry mapEntry; private String tag; private Object value; + private long tagId; void set(String tag, Object value) { this.mapEntry = null; this.tag = tag; this.value = value; + this.tagId = TagMap.TAG_ID_NOT_COMPUTED; // resolve lazily via keyOf on first tagId() access } void set(Map.Entry mapEntry) { this.mapEntry = mapEntry; this.tag = mapEntry.getKey(); this.value = mapEntry.getValue(); + this.tagId = TagMap.TAG_ID_NOT_COMPUTED; // resolve lazily via keyOf on first tagId() access } @Override @@ -2873,6 +2929,16 @@ public String tag() { return this.tag; } + @Override + public long tagId() { + long id = this.tagId; + if (id != TagMap.TAG_ID_NOT_COMPUTED) return id; + + id = KnownTagCodec.keyOf(this.tag); + this.tagId = id; + return id; + } + @Override public byte type() { return TagValueConversions.typeOf(this.value); diff --git a/internal-api/src/test/java/datadog/trace/api/EntryReadingHelperTest.java b/internal-api/src/test/java/datadog/trace/api/EntryReadingHelperTest.java index c47f15e1b3e..5da8564cb87 100644 --- a/internal-api/src/test/java/datadog/trace/api/EntryReadingHelperTest.java +++ b/internal-api/src/test/java/datadog/trace/api/EntryReadingHelperTest.java @@ -136,4 +136,34 @@ void setMapEntryCreatesEntry() { assertEquals("k", entry.tag()); assertEquals(99, entry.objectValue()); } + + @Test + void tagIdResolvesLazilyAndCaches() { + EntryReadingHelper helper = new EntryReadingHelper(); + helper.set("http.method", "GET"); + + long expectedId = KnownTagCodec.keyOf("http.method"); + assertEquals(expectedId, helper.tagId()); + // second call hits the already-resolved cache rather than re-resolving + assertEquals(expectedId, helper.tagId()); + } + + @Test + void tagIdOfUnknownTagResolvesToZeroAndCaches() { + EntryReadingHelper helper = new EntryReadingHelper(); + helper.set("my.custom.tag", "value"); + + assertEquals(0L, helper.tagId()); + assertEquals(0L, helper.tagId()); + } + + @Test + void tagIdIsRecomputedAfterSet() { + EntryReadingHelper helper = new EntryReadingHelper(); + helper.set("http.method", "GET"); + assertEquals(KnownTagCodec.keyOf("http.method"), helper.tagId()); + + helper.set(new AbstractMap.SimpleEntry<>("db.type", "mysql")); + assertEquals(KnownTagCodec.keyOf("db.type"), helper.tagId()); + } } diff --git a/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java new file mode 100644 index 00000000000..ecd9bb61e50 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java @@ -0,0 +1,227 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Checks the generated registry's name resolution and tag ID encoding. Datadog and OpenTelemetry + * names resolve to the same ID, while output names remain namespace-specific. + */ +class KnownTagsTest { + + /** (name, id) pairs across the groups — keyOf returns the id verbatim. */ + static Stream knownTags() { + return Stream.of( + Arguments.of(DDTags.PARENT_ID, KnownTags.DD_PARENT_ID), + Arguments.of(DDTags.BASE_SERVICE, KnownTags.DD_BASE_SERVICE_ID), + Arguments.of(Tags.VERSION, KnownTags.VERSION_ID), + Arguments.of("env", KnownTags.ENV_ID), + Arguments.of(DDTags.DJM_ENABLED, KnownTags.DD_DJM_ENABLED_ID), + Arguments.of(DDTags.DSM_ENABLED, KnownTags.DD_DSM_ENABLED_ID), + Arguments.of(DDTags.TRACER_HOST, KnownTags.DD_TRACER_HOST_ID), + Arguments.of(DDTags.DD_INTEGRATION, KnownTags.DD_INTEGRATION_ID), + Arguments.of(DDTags.DD_SVC_SRC, KnownTags.DD_SVC_SRC_ID), + Arguments.of(Tags.PEER_SERVICE, KnownTags.PEER_SERVICE_ID), + Arguments.of(DDTags.PEER_SERVICE_REMAPPED_FROM, KnownTags.DD_PEER_SERVICE_REMAPPED_FROM_ID), + Arguments.of(Tags.HTTP_METHOD, KnownTags.HTTP_METHOD_ID), + Arguments.of(Tags.HTTP_ROUTE, KnownTags.HTTP_ROUTE_ID), + Arguments.of(Tags.HTTP_URL, KnownTags.HTTP_URL_ID), + Arguments.of(Tags.PEER_HOSTNAME, KnownTags.PEER_HOSTNAME_ID), + Arguments.of(Tags.PEER_HOST_IPV4, KnownTags.PEER_IPV4_ID), + Arguments.of(Tags.PEER_HOST_IPV6, KnownTags.PEER_IPV6_ID), + Arguments.of(Tags.PEER_PORT, KnownTags.PEER_PORT_ID), + Arguments.of(Tags.COMPONENT, KnownTags.COMPONENT_ID), + Arguments.of(Tags.SPAN_KIND, KnownTags.SPAN_KIND_ID), + Arguments.of(DDTags.LANGUAGE_TAG_KEY, KnownTags.LANGUAGE_ID), + Arguments.of(Tags.DB_TYPE, KnownTags.DB_TYPE_ID), + Arguments.of(Tags.DB_INSTANCE, KnownTags.DB_INSTANCE_ID), + Arguments.of(Tags.DB_USER, KnownTags.DB_USER_ID), + Arguments.of(Tags.DB_OPERATION, KnownTags.DB_OPERATION_ID), + Arguments.of(Tags.DB_POOL_NAME, KnownTags.DB_POOL_NAME_ID)); + } + + /** + * (otelName, canonicalId, datadogName) — the OpenTelemetry name resolves (keyOf) to the canonical + * tag's id; datadogNameOf returns the Datadog name and openTelemetryNameOf returns the OTel name. + */ + static Stream otelNamedTags() { + return Stream.of( + Arguments.of("http.request.method", KnownTags.HTTP_METHOD_ID, "http.method"), + Arguments.of( + "http.response.status_code", KnownTags.HTTP_STATUS_CODE_ID, "http.status_code"), + Arguments.of("url.full", KnownTags.HTTP_URL_ID, "http.url"), + Arguments.of("user_agent.original", KnownTags.HTTP_USERAGENT_ID, "http.useragent"), + Arguments.of("client.address", KnownTags.HTTP_CLIENT_IP_ID, "http.client_ip"), + Arguments.of("db.system", KnownTags.DB_TYPE_ID, "db.type"), + Arguments.of("db.operation.name", KnownTags.DB_OPERATION_ID, "db.operation"), + Arguments.of("db.query.text", KnownTags.DB_STATEMENT_ID, "db.statement"), + Arguments.of("service.name", KnownTags.SERVICE_ID, "service")); + } + + /** + * Trace-level tags (live on the TraceSegment's TagMap) — their id carries the LEVEL_TRACE bit. + */ + static Stream traceLevelTags() { + return Stream.of( + Arguments.of(KnownTags.DD_BASE_SERVICE_ID), + Arguments.of(KnownTags.VERSION_ID), + Arguments.of(KnownTags.ENV_ID), + Arguments.of(KnownTags.LANGUAGE_ID), + Arguments.of(KnownTags.RUNTIME_ID), + Arguments.of(KnownTags.DD_TRACER_HOST_ID), + Arguments.of(KnownTags.DD_DJM_ENABLED_ID)); + } + + /** Span-level tags — their id leaves the LEVEL_TRACE bit clear. */ + static Stream spanLevelTags() { + return Stream.of( + Arguments.of(KnownTags.HTTP_METHOD_ID), + Arguments.of(KnownTags.HTTP_URL_ID), + Arguments.of(KnownTags.DB_TYPE_ID), + Arguments.of(KnownTags.COMPONENT_ID), + Arguments.of(KnownTags.SPAN_KIND_ID), + Arguments.of(KnownTags.PEER_SERVICE_ID)); + } + + @ParameterizedTest + @MethodSource("knownTags") + void keyOfResolvesNameToId(String name, long id) { + assertEquals(id, KnownTagCodec.keyOf(name), "keyOf(" + name + ")"); + } + + @ParameterizedTest + @MethodSource("knownTags") + void nameOfResolvesIdToName(String name, long id) { + assertEquals(name, KnownTagCodec.nameOf(id), "nameOf(" + name + ")"); + } + + @ParameterizedTest + @MethodSource("otelNamedTags") + void otelNameResolvesToCanonicalId(String otelName, long id, String datadogName) { + // Inbound (keyOf) is many->one: both names land on the same canonical id. + assertEquals(id, KnownTagCodec.keyOf(otelName), "keyOf(" + otelName + ")"); + assertEquals(id, KnownTagCodec.keyOf(datadogName), "keyOf(" + datadogName + ")"); + } + + @ParameterizedTest + @MethodSource("otelNamedTags") + void namespaceAccessorsReturnPerNamespaceName(String otelName, long id, String datadogName) { + assertEquals(datadogName, KnownTagCodec.datadogNameOf(id), "datadogNameOf"); + assertEquals(otelName, KnownTagCodec.openTelemetryNameOf(id), "openTelemetryNameOf"); + // nameOf stays the Datadog name -- outbound is namespace-specific, not normalized to OTel. + assertEquals(datadogName, KnownTagCodec.nameOf(id), "nameOf stays Datadog"); + } + + @ParameterizedTest + @MethodSource("otelNamedTags") + void openTelemetryTagOfReturnsTheRename(String otelName, long id, String datadogName) { + assertEquals(otelName, KnownTagCodec.openTelemetryTagOf(id), "openTelemetryTagOf"); + } + + @Test + void openTelemetryTagOfPassesThroughWhenThereIsNoRename() { + // http.route declares no otel-name, so the OpenTelemetry namespace emits the Datadog name. + assertNull(KnownTagCodec.openTelemetryNameOf(KnownTags.HTTP_ROUTE_ID), "no declared rename"); + assertEquals( + KnownTagCodec.nameOf(KnownTags.HTTP_ROUTE_ID), + KnownTagCodec.openTelemetryTagOf(KnownTags.HTTP_ROUTE_ID), + "pass-through falls back to the Datadog name"); + } + + @Test + void openTelemetryTagOfIsNullForAnUnknownId() { + // A custom tag has no registry name in any namespace; only its holder knows its key. + assertNull(KnownTagCodec.openTelemetryTagOf(0L)); + } + + @Test + void tagsWithoutOtelNameReturnNull() { + assertNull(KnownTagCodec.openTelemetryNameOf(KnownTags.HTTP_ROUTE_ID)); // no OTel name declared + assertNull(KnownTagCodec.openTelemetryNameOf(0L)); // unknown id + } + + @Test + void unknownNamesResolveToZero() { + assertEquals(0L, KnownTagCodec.keyOf("definitely.not.a.known.tag")); + assertEquals(0L, KnownTagCodec.keyOf("http.statuscode")); // close-but-not-listed + assertEquals(0L, KnownTagCodec.keyOf("")); + } + + @Test + void unknownIdsResolveToNullName() { + assertNull(KnownTagCodec.nameOf(0L)); + assertNull(KnownTagCodec.nameOf(KnownTagCodec.makeTagId(9999))); // serial with no assigned tag + } + + /** + * A mixin declares tags for the span types its {@code applies:} names. {@code ci_visibility} + * applies to {@code test}, which the conventions do not model yet -- so these tags belong to no + * concrete type's resolved set. They must still be registered: an id is identity, and identity + * does not depend on layout. Building the registry by resolving concrete types instead dropped + * all four silently, leaving keyOf to report live CI Visibility tags as unknown. + */ + @ParameterizedTest + @MethodSource("declarationOnlyMixinTags") + void mixinTagsAreRegisteredEvenWhenTheirSpanTypeIsNotModeled(String name, long id) { + assertEquals(id, KnownTagCodec.keyOf(name), "keyOf(" + name + ")"); + assertEquals(name, KnownTagCodec.nameOf(id), "nameOf(" + name + ")"); + } + + /** Tags reachable only through a mixin whose {@code applies:} target is not modeled. */ + static Stream declarationOnlyMixinTags() { + return Stream.of( + Arguments.of(KnownTags.TEST_NAME, KnownTags.TEST_NAME_ID), + Arguments.of(KnownTags.TEST_SUITE_NAME, KnownTags.TEST_SUITE_ID), + Arguments.of(KnownTags.TEST_STATUS_NAME, KnownTags.TEST_STATUS_ID), + Arguments.of(KnownTags.TEST_FRAMEWORK_NAME, KnownTags.TEST_FRAMEWORK_ID)); + } + + @Test + void globalSerialsAreUnique() { + List serials = new ArrayList<>(); + knownTags().forEach(a -> serials.add((long) KnownTagCodec.serialNum((Long) a.get()[1]))); + assertEquals(serials.size(), new HashSet<>(serials).size(), "globalSerials must be unique"); + } + + @ParameterizedTest + @MethodSource("traceLevelTags") + void traceLevelTagsCarryLevelBit(long id) { + assertTrue(KnownTagCodec.isTraceLevel(id), "isTraceLevel"); + } + + @ParameterizedTest + @MethodSource("spanLevelTags") + void spanLevelTagsClearLevelBit(long id) { + assertFalse(KnownTagCodec.isTraceLevel(id), "not trace-level"); + } + + @Test + void levelBitCompositionRoundTrips() { + long spanId = KnownTagCodec.makeTagId(300); // no level bit + assertFalse(KnownTagCodec.isTraceLevel(spanId)); + long traceId = KnownTagCodec.traceLevel(spanId); + assertTrue(KnownTagCodec.isTraceLevel(traceId)); + // level bit is orthogonal to the serial — it survives setting the bit + assertEquals(KnownTagCodec.serialNum(spanId), KnownTagCodec.serialNum(traceId)); + assertEquals(traceId, KnownTagCodec.traceLevel(traceId), "traceLevel is idempotent"); + } + + @Test + void serialEncodingRoundTrips() { + long id = KnownTagCodec.makeTagId(263); + assertEquals(263, KnownTagCodec.serialNum(id)); + assertFalse(KnownTagCodec.isTraceLevel(id)); + } +} diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapNamespaceNamesTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapNamespaceNamesTest.java new file mode 100644 index 00000000000..1776f32c291 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/TagMapNamespaceNamesTest.java @@ -0,0 +1,81 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * {@link TagMap.EntryReader#openTelemetryTag()} — the reader's view of its tag in the OpenTelemetry + * namespace. {@link KnownTagCodec#openTelemetryTagOf} owns the naming and returns null for a tag it + * does not know; the reader completes that by falling back to its own key, which is the one thing + * the codec cannot do. + */ +class TagMapNamespaceNamesTest { + + @Test + void renamedTagReadsUnderItsOpenTelemetryName() { + assertEquals("http.request.method", otelTagOf("http.method", "GET")); + } + + @Test + void tagWithoutARenamePassesThroughUnderItsDatadogName() { + // http.route declares no otel-name, so the OpenTelemetry namespace keeps the Datadog spelling. + assertEquals("http.route", otelTagOf("http.route", "/orders/:id")); + } + + @Test + void customTagFallsBackToItsOwnKey() { + // Not in the registry at all: the codec has no name for it, so the reader supplies its key. + assertEquals("my.app.tenant", otelTagOf("my.app.tenant", "acme")); + } + + @Test + void anOpenTelemetrySpellingNormalizesToTheOneName() { + // keyOf is many->one, so an entry written under the OTel name resolves to the same tag and + // reads back under that name -- not under two different ones depending on how it was written. + assertEquals("http.request.method", otelTagOf("http.request.method", "POST")); + assertEquals( + otelTagOf("http.method", "GET"), + otelTagOf("http.request.method", "POST"), + "both spellings of one tag must read under the same OpenTelemetry name"); + } + + @Test + void openTelemetryNameDiffersFromTheDatadogNameForARenamedTag() { + TagMap map = TagMap.create(); + map.set("http.method", "GET"); + TagMap.EntryReader reader = readerFor(map, "http.method"); + assertEquals("http.method", reader.tag(), "tag() stays the key as written"); + assertNotEquals( + reader.tag(), reader.openTelemetryTag(), "a rename must actually change the emitted name"); + } + + private static String otelTagOf(String tag, Object value) { + TagMap map = TagMap.create(); + map.set(tag, value); + return readerFor(map, tag).openTelemetryTag(); + } + + /** + * The entry for {@code tag}, having first checked that iteration agrees with it — the iterator + * may hand out a reused flyweight rather than the entry itself, so the two paths are worth + * pinning together. + */ + private static TagMap.EntryReader readerFor(TagMap map, String tag) { + Map otelByTag = new HashMap<>(); + map.forEach(reader -> otelByTag.put(reader.tag(), reader.openTelemetryTag())); + + TagMap.Entry entry = map.getEntry(tag); + // Key off the entry's own (canonical) tag(), not the possibly-pre-normalization `tag` + // argument: entries are stored under their canonical Datadog name, so a lookup by an + // OpenTelemetry alias returns an entry whose tag() differs from the string used to set it. + assertEquals( + otelByTag.get(entry.tag()), + entry.openTelemetryTag(), + "iteration and getEntry must agree on the OpenTelemetry name for " + tag); + return entry; + } +} diff --git a/tag-conventions.yaml b/tag-conventions.yaml new file mode 100644 index 00000000000..f35498e980b --- /dev/null +++ b/tag-conventions.yaml @@ -0,0 +1,169 @@ +# Tag conventions — LANGUAGE-AGNOSTIC domain spec (structure + semantics only) +# --------------------------------------------------------------------------- +# The code generator consumes THIS file to emit each language's tag-id constants and its +# id<->name resolver. A tag id is IDENTITY (a globally unique serial + the trace-level bit); +# storage layout (the dense store's slot assignment) and set-path routing (which keys the tracer +# intercepts into span fields or sampling directives) are per-language concerns that arrive with +# the code that consumes them, via a per-language overlay alongside this file. +# +# TRACE-LEVEL is its own thing (its own TagMap "type" on the TraceSegment) — the process/trace +# constants + product flags that are set once per trace, NOT per span. Declared explicitly in the +# `trace_level` section below (a distinct tier), never inferred from `source`. +# +# SPAN TYPES compose three ways: +# extends — structural is-a inheritance (http.server is-a http is-a base). `base` is implicitly +# in every span; abstract layers exist only to be extended. +# include — a span type PULLS in a mixin it intrinsically has (has-a; core-owned). +# applies — a mixin PUSHES itself onto span types, gated by `enabled_by`. +# resolved_tags(type) = own + extends-chain (incl base) + included mixins + applied mixins (de-duped). +# +# tag fields (DOMAIN only): dd-name | type (string|int|long|boolean|double) +# | required (required|conditional|recommended|optional|opt_in) | otel-name. +# dd-name is the canonical Datadog-namespace name AND the tag's identity. otel-name is OPTIONAL and +# tri-state: +# - absent => the OpenTelemetry name is IMPLICITLY the dd-name (the tag passes through under +# its Datadog name; this is the RFC "retain" default for tags with no rename). +# - a name => rename: the tag is emitted under that OpenTelemetry-namespace name instead. +# - the literal none => Datadog-only: the tag has NO OpenTelemetry name (suppressed from OTel). This +# value is reserved — no tag uses it today (the RFC renames or retains, never +# suppresses), and real suppression is a follow-on; it currently behaves as +# pass-through. +# A tag is one identity across span types/mixins, so it is DECLARED exactly once — a second declaration +# fails the build. Put a tag shared by several span types on their common parent or a mixin; any other +# span type that carries it uses `{ ref: , required: }`, which may override only +# `required` (type and otel-name come from the one declaration). A span-kind-dependent mapping is a +# derivation, not a rename. +# An otel-name is canonicalized to its tag on EVERY span kind, so a rename declared on a concrete span +# type must also set `span-kind-neutral: true`: the author's assertion that the OpenTelemetry attribute +# means this tag wherever OTel uses it. (Renames in a shared scope -- an abstract parent or a mixin -- +# need no flag.) OTel's server.address and network.peer.address fail that test: on a client span they +# name the remote server. +# The id coordinate (group-decl / field-decl) is NOT authored here — the generator assigns it: each +# declaration source (the trace-level tier, each span type, each mixin) is a group, and within a +# group `field-decl` numbers the dense (required/conditional/recommended) tags; the rest are +# bucketed. See the design doc. +# --------------------------------------------------------------------------- + +# Trace-level tier: its own TagMap on the TraceSegment. Set once per trace, not per span. +# (Their OTel mapping is a resource-attribute follow-on; they pass through under dd-name for now.) +trace_level: + tags: + - { dd-name: _dd.base_service, type: string, required: required } + - { dd-name: version, type: string, required: recommended } + - { dd-name: env, type: string, required: recommended } + - { dd-name: language, type: string, required: required } + - { dd-name: runtime-id, type: string, required: required } + - { dd-name: _dd.tracer_host, type: string, required: recommended } + - { dd-name: _dd.git.commit.sha, type: string, required: recommended } + - { dd-name: _dd.git.repository_url, type: string, required: recommended } + # product .enabled flags — process-constant; present on the trace segment regardless of whether + # the product is enabled (the flag carries the state), so always-present => recommended. + - { dd-name: _dd.profiling.enabled, type: boolean, required: recommended } + - { dd-name: _dd.dsm.enabled, type: boolean, required: recommended } + - { dd-name: _dd.appsec.enabled, type: boolean, required: recommended } + - { dd-name: _dd.djm.enabled, type: boolean, required: recommended } + - { dd-name: _dd.civisibility.enabled, type: boolean, required: recommended } + +span_types: + # root: per-span tags every span has (incl. the per-span core tags parent_id / integration / svc_src + # — core-set but per-span, so NOT trace-level). + base: + abstract: true + tags: + - { dd-name: _dd.parent_id, type: string, required: required } + - { dd-name: service, type: string, required: required, otel-name: service.name } + - { dd-name: component, type: string, required: required } + - { dd-name: span.kind, type: string, required: required } # OTel span kind is a first-class field, not an attribute + - { dd-name: _dd.integration, type: string, required: recommended } + - { dd-name: _dd.svc_src, type: string, required: optional } + - { dd-name: error.type, type: string, required: recommended } # TODO(otel): map error.* to exception.* semconv + - { dd-name: error.message, type: string, required: recommended } + - { dd-name: error.stack, type: string, required: recommended } + + http: + abstract: true + extends: base + tags: + - { dd-name: http.method, type: string, required: required, otel-name: http.request.method } + - { dd-name: http.status_code, type: int, required: conditional, otel-name: http.response.status_code } + - { dd-name: network.protocol.version, type: string, required: recommended } # passes through: dd-name already is the OTel name + - { dd-name: http.url, type: string, required: required, otel-name: url.full } # shared by http.server and http.client => one otel-name. url.full is the client-correct rename; server's spec mapping (url.path + url.scheme + url.query) is a one-to-many split reserved for the derivation layer (needs span.kind). TODO(otel): server split. + + http.server: + extends: http + tags: + - { dd-name: http.route, type: string, required: conditional } # passes through: dd-name already is the OTel name + - { dd-name: http.hostname, type: string, required: required } # not a rename: OTel server.address is the remote server on client spans; needs the span-kind-aware derivation layer + - { dd-name: http.useragent, type: string, required: recommended, otel-name: user_agent.original, span-kind-neutral: true } + - { dd-name: http.query.string, type: string, required: recommended } # not a rename: url.query is also carried inside url.full, while http.url excludes the query (QueryObfuscator re-appends it) + - { dd-name: servlet.path, type: string, required: optional } + - { dd-name: servlet.context, type: string, required: optional } + - { dd-name: http.client_ip, type: string, required: recommended, otel-name: client.address, span-kind-neutral: true } + - { dd-name: network.client.ip, type: string, required: recommended } # not a rename: OTel network.peer.address is the remote end on every span kind (the server on client spans) + + http.client: + extends: http + include: [ peer ] + tags: + - { dd-name: http.resend_count, type: int, required: recommended } + + db.client: + extends: base + include: [ peer ] + tags: + - { dd-name: db.type, type: string, required: required, otel-name: db.system, span-kind-neutral: true } + - { dd-name: db.instance, type: string, required: recommended } # TODO(otel): db.namespace + - { dd-name: db.operation, type: string, required: recommended, otel-name: db.operation.name, span-kind-neutral: true } + - { dd-name: db.user, type: string, required: recommended } + - { dd-name: db.pool.name, type: string, required: optional } + - { dd-name: db.statement, type: string, required: recommended, otel-name: db.query.text, span-kind-neutral: true } + + view.render: + extends: base + tags: + - { dd-name: view.name, type: string, required: recommended } + +mixins: + # peer — outbound/remote-peer capability, PULLED via `include` by client span types. + peer: + tags: + - { dd-name: peer.service, type: string, required: recommended } + - { dd-name: _dd.peer.service.source, type: string, required: recommended } + - { dd-name: _dd.peer.service.remapped_from, type: string, required: recommended } + - { dd-name: peer.hostname, type: string, required: recommended } + - { dd-name: peer.ipv4, type: string } + - { dd-name: peer.ipv6, type: string } + - { dd-name: peer.port, type: int } + + # ci_visibility — per-span test tags. Its capability flag (_dd.civisibility.enabled) lives in + # trace_level, outside this mixin (general rule: capability flags are trace-level, mixins hold the + # per-span tags). Applies to the `test` span type, which is not modeled here yet — so these tags + # get ids (identity does not depend on layout) but contribute to no type's resolved set until it + # is. The generator reports that gap in resolved-tags.txt. + ci_visibility: + enabled_by: dd.civisibility.enabled + applies: [ test ] + tags: + - { dd-name: test.name, type: string, required: recommended } + - { dd-name: test.suite, type: string, required: recommended } + - { dd-name: test.status, type: string, required: recommended } + - { dd-name: test.framework, type: string, required: recommended } + +# --------------------------------------------------------------------------- +# Notes +# - Product .enabled flags moved to `trace_level` (process-constant) — the old product mixins held +# only those flags, so they dissolved. `enabled_by`/attachment gating is a runtime concern. +# - span.kind enumerates: server | client | producer | consumer | internal | broker. +# - Some keys (resource.name, error, sampling.priority, ...) are accepted by setTag but routed to a +# span field or a trace directive instead of tag storage. That routing is a per-language tracer +# concern, so it is NOT modelled here; such a key appears above only when it also needs an id and +# a name (service does, for OpenTelemetry's service.name). +# - Tags with no otel-name pass through under their Datadog name (RFC "retain"). A `# TODO(otel)` note +# marks a pending OpenTelemetry-team review of a mapping that is not yet a settled rename. +# - db.statement/db.query.text is deliberately never re-exposed as a stored tag once consumed into +# resource.name, regardless of which alias set it -- the two names are equivalent for identity/id +# purposes, but the raw value (often unobfuscated SQL) is intentionally not retained. This is an +# open question (not yet resolved one way or the other): revisit only as a language-agnostic policy +# change (i.e. decide to start storing/exporting it in every tracer), not as a per-language or +# per-spelling special case. +# ---------------------------------------------------------------------------