From 3a309eed47e261625152e091b83e1ac56508f2f4 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 14 Oct 2026 21:31:36 -0400 Subject: [PATCH 1/6] Color known tags into co-occurrence slots in their ids The tag-registry generator now assigns each known tag a slot in id bits [47-32] by graph coloring: each concrete span type's resolved tags, and the trace-level tags, form a clique and get distinct slots. So 1L << slotOf(id) is an exact presence bit for a span's tags, and one long holds them all. Tags on no modeled span type get NO_SLOT, which a long shift maps onto bit 63; real slots are capped below it. KnownTagCodec gains SLOT_SHIFT, NO_SLOT, slotOf, hasSlot and slotCount. Nothing in the tracer reads slots yet: this is split out of the dense store (#12856) so the store, the span builder's tag ledger and a map-level routing summary can each build on it. Co-Authored-By: Claude Opus 5.5 --- .../tagRegistry/KnownTagsEmitter.kt | 4 + .../buildlogic/tagRegistry/TagRegistry.kt | 74 +++++++++++++++--- .../tagRegistry/TagRegistryGenerator.kt | 5 +- .../tagRegistry/TagRegistryGeneratorTest.kt | 44 ++++++++++- .../java/datadog/trace/api/KnownTagCodec.java | 40 ++++++++-- .../trace/api/KnownTagCodecSlotTest.java | 77 +++++++++++++++++++ 6 files changed, 221 insertions(+), 23 deletions(-) create mode 100644 internal-api/src/test/java/datadog/trace/api/KnownTagCodecSlotTest.java diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt index a68d39d6701..63715868802 100644 --- a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt @@ -83,6 +83,7 @@ object KnownTagsEmitter { appendLine(" public static final String ${otelNameC(t.ddName)} = \"${escape(t.otelName)}\";") } append("// makeTagId(serial=${t.serial})") + append(if (t.slot == TagRegistry.NO_SLOT) " + no slot" else " + slot=${t.slot}") if (t.traceLevel) append(" + trace-level") if (t.intercepted) append(" + intercepted") if (t.otelName != null) append(" -> ${escape(t.otelName)}") @@ -100,6 +101,9 @@ object KnownTagsEmitter { // Every known tag's serial is below SERIAL_LIMIT; serial 0 is no tag. val serialLimit = (reg.tags.maxOfOrNull { it.serial } ?: 0) + 1 appendLine(" public static final int SERIAL_LIMIT = $serialLimit;") + // Slots in use: every slotted tag's slot is below SLOT_COUNT. + val slotCount = (reg.tags.filter { it.slot != TagRegistry.NO_SLOT }.maxOfOrNull { it.slot } ?: -1) + 1 + appendLine(" public static final int SLOT_COUNT = $slotCount;") // OpenTelemetry name -> canonical tag name. Validation ensures aliases are distinct from all // canonical names. Sort by OTel name to keep output deterministic. diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt index 248471c228a..cf1eb3e572b 100644 --- a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt @@ -2,12 +2,20 @@ package datadog.buildlogic.tagRegistry /** * Assigns tag ids from a parsed [TagConventions]. The id encoding mirrors KnownTagCodec: [63-48 - * serial][47-32 reserved][31-0 flags]. + * serial][47-32 slot][31-0 flags]. * - *

An id is a globally unique serial plus two classification bits: trace-level and intercepted. It - * carries no storage-layout coordinate -- bits [47-32] are held vacant for the co-occurrence slot - * that the dense tag store assigns by graph coloring, which lands with the dense store itself. - * Nothing here needs to know how (or whether) a tag is stored. + *

An id is a globally unique serial, a slot, and two classification bits: trace-level and + * intercepted. + * + *

The slot comes from GRAPH COLORING the tag co-occurrence graph. Each concrete span type's + * resolved tags appear together on one span, so they form a clique and need distinct slots; the + * trace-level tier is its own clique (it lives in a separate TagMap). Tags that never share a clique + * may share a slot, so the slot count stays bounded by the largest clique, small enough for one + * occupancy `long`. Every tag resolved onto a span type, or trace-level, is slotted whatever its + * requirement level; a tag on no modeled span type (the tracer overlay's routing-only keys, tags of + * unmodeled span types) gets [NO_SLOT]. Nothing in the tracer reads slots yet: they are for + * per-span structures that want an exact, compact presence word, such as a dense tag store or a + * span builder's tag ledger. * *

The intercepted bit marks a tag the tracer may route on the set-path (to a span field or a * sampling directive instead of tag storage), as listed by the tracer overlay. A setter called with @@ -26,6 +34,8 @@ class TagRegistry private constructor(val tags: List) { val serial: Int, val traceLevel: Boolean, val intercepted: Boolean, + /** The tag's slot, or [NO_SLOT] for a tag on no modeled span type. */ + val slot: Int, val id: Long, /** The tag's OpenTelemetry name in every direction; null when not renamed, or renamed per direction. */ val otelName: String? = null, @@ -42,13 +52,17 @@ class TagRegistry private constructor(val tags: List) { const val INTERCEPTED = 1L shl 1 // low-32 carve bit 1; mirrors KnownTagCodec.INTERCEPTED const val LEVEL_TRACE = 1L shl 2 // low-32 carve bit 2; mirrors KnownTagCodec.LEVEL_TRACE + const val SLOT_SHIFT = 32 // slot window [47-32]; mirrors KnownTagCodec.SLOT_SHIFT + const val NO_SLOT = 0xFFFF // all-ones slot: no modeled span type; mirrors KnownTagCodec.NO_SLOT + const val MAX_SLOTS = 63 // fits one occupancy long; bit 63 is left for unslotted tags to share + /** - * Mirrors KnownTagCodec.makeTagId(serial) + intercepted() + traceLevel() -- must stay in sync. - * INTERCEPTED at bit 1, LEVEL_TRACE at bit 2, other low bits and the reserved [47-32] window - * zero. + * Mirrors KnownTagCodec's layout -- must stay in sync: serial at [63-48], slot at [47-32], + * INTERCEPTED at bit 1, LEVEL_TRACE at bit 2, other low bits zero. */ - fun encode(serial: Int, traceLevel: Boolean, intercepted: Boolean): Long { - var id = serial.toLong() shl 48 + fun encode(serial: Int, traceLevel: Boolean, intercepted: Boolean, slot: Int = NO_SLOT): Long { + require(slot == NO_SLOT || slot in 0 until MAX_SLOTS) { "slot $slot is outside [0, $MAX_SLOTS)" } + var id = (serial.toLong() shl 48) or (slot.toLong() shl SLOT_SHIFT) if (traceLevel) id = id or LEVEL_TRACE if (intercepted) id = id or INTERCEPTED return id @@ -57,6 +71,7 @@ class TagRegistry private constructor(val tags: List) { fun build(conv: TagConventions): TagRegistry { val traceLevel = conv.traceLevelTags().map { it.ddName }.toSet() val renames = conv.otelMappings().groupBy { it.tag.ddName } + val slots = colorSlots(conv) // One tag per Datadog name, in a stable order (by name) so serials -- and therefore ids -- are // a pure function of the input. A name's declarations per direction merge into its one tag. @@ -70,6 +85,7 @@ class TagRegistry private constructor(val tags: List) { val isTraceLevel = first.ddName in traceLevel val isIntercepted = first.ddName in conv.interceptedNames val (otelName, scoped) = otelNamesOf(first.ddName, renames[first.ddName].orEmpty()) + val slot = slots[first.ddName] ?: NO_SLOT Tag( first.ddName, first.type, @@ -77,7 +93,8 @@ class TagRegistry private constructor(val tags: List) { serial, isTraceLevel, isIntercepted, - id = encode(serial, isTraceLevel, isIntercepted), + slot, + id = encode(serial, isTraceLevel, isIntercepted, slot), otelName = otelName, scopedOtelNames = scoped, ) @@ -87,6 +104,41 @@ class TagRegistry private constructor(val tags: List) { return TagRegistry(tags) } + /** + * Colors the co-occurrence graph of slotted tags: tags resolved onto one concrete span type, or + * both trace-level, must get distinct slots. Greedy, most-constrained first (by clique count, + * then name), so the result is a pure function of the conventions. A name declared per direction + * is one tag, so it is one vertex. + */ + private fun colorSlots(conv: TagConventions): Map { + val cliques = ArrayList>() + for (type in conv.concreteTypes()) { + cliques.add(conv.resolve(type).map { it.ddName }.toSet()) + } + cliques.add(conv.traceLevelTags().map { it.ddName }.toSet()) + + val neighbors = HashMap>() + for (clique in cliques) { + for (a in clique) { + val adjacent = neighbors.getOrPut(a) { HashSet() } + for (b in clique) if (a != b) adjacent.add(b) + } + } + val cliqueCount = neighbors.keys.associateWith { tag -> cliques.count { tag in it } } + val order = neighbors.keys.sortedWith(compareByDescending { cliqueCount.getValue(it) }.thenBy { it }) + val slots = HashMap() + for (tag in order) { + val taken = neighbors.getValue(tag).mapNotNull { slots[it] }.toSet() + var slot = 0 + while (slot in taken) slot++ + require(slot < MAX_SLOTS) { + "coloring needs more than $MAX_SLOTS slots at '$tag'; one occupancy long holds $MAX_SLOTS" + } + slots[tag] = slot + } + return slots + } + /** * A tag's renames, as one name for every direction or one name per direction. The rename of a * declaration made per direction applies in that declaration's direction. diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt index eaae7ccc1a6..3baf2214638 100644 --- a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt @@ -70,16 +70,17 @@ object TagRegistryGenerator { """ # Tag id assignment. tags=${reg.tags.size} - # TAGS serial lvl int id required name + # TAGS serial lvl int slot id required name """.trimIndent() ) for (t in reg.tags) { appendLine( - " %6d %s %s %-18s %-12s %s".format( + " %6d %s %s %4s %-18s %-12s %s".format( Locale.ROOT, t.serial, if (t.traceLevel) "T" else "-", if (t.intercepted) "I" else "-", + if (t.slot == TagRegistry.NO_SLOT) "-" else t.slot.toString(), "0x%016X".format(Locale.ROOT, t.id), t.required, t.name diff --git a/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt index a88f7509144..6db58e6f27b 100644 --- a/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt +++ b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt @@ -535,9 +535,9 @@ class TagRegistryGeneratorTest { val knownTags = contents(output).getValue("java/datadog/trace/api/KnownTags.java") assertThat(knownTags) .contains( - "COMPONENT_ID = 0x0001000000000000L", - "RESOURCE_NAME_ID = 0x0002000000000002L", - "SERVICE_ID = 0x0003000000000002L", + "COMPONENT_ID = 0x0001FFFF00000000L", + "RESOURCE_NAME_ID = 0x0002FFFF00000002L", + "SERVICE_ID = 0x0003FFFF00000002L", "public static final int SERVICE_SERIAL_NUM = 3;", "public static final int SERIAL_LIMIT = 4;" ) @@ -561,6 +561,44 @@ class TagRegistryGeneratorTest { .withMessageContaining(message) } + @Test + fun `tags on one span type get distinct slots and tags that never co-occur share one`() { + val yaml = directory.conventionsFile( + """ + trace_level: + tags: [{dd-name: env, required: required}] + span_types: + base: + abstract: true + tags: [{dd-name: service, required: required}] + http.server: + extends: base + span-kind: server + tags: [{dd-name: http.route, required: required}] + db.client: + extends: base + span-kind: client + tags: [{dd-name: db.type, required: required}, {dd-name: db.user}] + """ + ) + val output = File(directory, "generated") + + TagRegistryGenerator.generate(yaml, output) + + // service is on both span types, so it takes slot 0; db.type and db.user share db.client, so + // they take distinct slots, and http.route -- never on a span type with either -- reuses one of + // them; env is trace-level (its own TagMap) and reuses slot 0. Every level is slotted. + assertThat(contents(output).getValue("java/datadog/trace/api/KnownTags.java")) + .contains( + "DB_TYPE_ID = 0x0001000100000000L", + "DB_USER_ID = 0x0002000200000000L", + "ENV_ID = 0x0003000000000004L", + "HTTP_ROUTE_ID = 0x0004000100000000L", + "SERVICE_ID = 0x0005000000000000L", + "public static final int SLOT_COUNT = 3;" + ) + } + private fun tagConventions(@Language("yaml") yamlText: String) = TagConventions.parse( ObjectMapper(YAMLFactory()).readValue( directory.conventionsFile(yamlText), diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java index 605f7140d4c..6e09e4d1aad 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -13,11 +13,11 @@ */ public final class KnownTagCodec { /* - * tagId bit layout: [63-48 serialNum (16 bits)] [47-32 reserved, zero] [31-0 flags]. serialNum is + * tagId bit layout: [63-48 serialNum (16 bits)] [47-32 slot] [31-0 flags]. serialNum is * globally unique per known tag and is the whole of the tag's identity — nameOf/ * openTelemetryNameOf switch on it, and the generator emits each id as a literal. Bits [47-32] - * are RESERVED and always zero here: they are the window the dense tag store uses for its - * co-occurrence slot coordinate, which arrives with that store. Of the low 32 flag bits, bit 2 is + * hold the tag's slot ({@link #slotOf}), assigned by graph coloring so tags on one span type never + * share a slot, or {@link #NO_SLOT} for a tag on no modeled span type. Of the low 32 flag bits, bit 2 is * the trace/span LEVEL bit (set ⟹ trace-level), and bit 1 is the INTERCEPTED bit. Unknown * (string-only) custom tags are NOT known ids — {@code keyOf} returns 0 for them. * @@ -65,16 +65,42 @@ public static boolean isIntercepted(long tagId) { return (tagId & INTERCEPTED) != 0L; } + /** Bit position of the slot: bits [47-32] of an id. */ + public static final int SLOT_SHIFT = 32; + + /** The slot of a known tag on no modeled span type. */ + public static final int NO_SLOT = 0xFFFF; + + /** + * The tag's slot, or {@link #NO_SLOT}. The generator colors tags that appear together on a span + * type with distinct slots, so a span's slotted tags never collide: {@code 1L << slotOf(id)} is + * an exact presence bit for them. Every tag without a slot shares bit 63, which no real slot + * uses. + */ + public static int slotOf(long tagId) { + return (int) (tagId >>> SLOT_SHIFT) & 0xFFFF; + } + + /** True if the tag has a slot, below {@link #slotCount()}. */ + public static boolean hasSlot(long tagId) { + return slotOf(tagId) != NO_SLOT; + } + + /** Slots in use: every slotted tag's slot is below this. */ + public static int slotCount() { + return KnownTags.SLOT_COUNT; + } + /** Returns the tagId with the {@link #LEVEL_TRACE} flag set. */ public static long traceLevel(long tagId) { return tagId | LEVEL_TRACE; } /** - * Builds a tagId from its {@code serialNum} (globally unique per known tag). The reserved [47-32] - * window and the low 32 bits are zero, so the id is fully determined by the serial — the - * generator emits it as a literal. Inverse of {@link #serialNum}. Intended for the code generator - * and tests. + * Builds an id carrying only {@code serialNum}: slot 0 and no flags, so it equals a tag's real id + * only when that tag has neither. Enough for serial-keyed lookups such as {@link #nameOf}; + * compare against a {@code KnownTags} constant, never against this. Inverse of {@link + * #serialNum}. Intended for tests and serial-keyed lookups. */ public static long makeTagId(int serialNum) { return (long) serialNum << 48; diff --git a/internal-api/src/test/java/datadog/trace/api/KnownTagCodecSlotTest.java b/internal-api/src/test/java/datadog/trace/api/KnownTagCodecSlotTest.java new file mode 100644 index 00000000000..4dd09044be8 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/KnownTagCodecSlotTest.java @@ -0,0 +1,77 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Field; +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.Test; + +/** {@link KnownTagCodec}'s slot: bits [47-32] of an id, assigned by graph coloring. */ +class KnownTagCodecSlotTest { + @Test + void everySlotIsBelowTheSlotCount() throws IllegalAccessException { + for (Field field : KnownTags.class.getFields()) { + if (field.getType() != long.class) { + continue; + } + long tagId = field.getLong(null); + if (KnownTagCodec.hasSlot(tagId)) { + assertTrue(KnownTagCodec.slotOf(tagId) < KnownTagCodec.slotCount(), field.getName()); + } else { + assertEquals(KnownTagCodec.NO_SLOT, KnownTagCodec.slotOf(tagId), field.getName()); + } + } + } + + @Test + void tagsOnAnHttpServerSpanHaveDistinctSlots() { + long[] httpServerTags = { + KnownTags.SERVICE_ID, + KnownTags.COMPONENT_ID, + KnownTags.SPAN_KIND_ID, + KnownTags.HTTP_METHOD_ID, + KnownTags.HTTP_URL_ID, + KnownTags.HTTP_STATUS_CODE_ID, + KnownTags.HTTP_ROUTE_ID, + KnownTags.HTTP_HOSTNAME_ID, + }; + Set slots = new HashSet<>(); + for (long tagId : httpServerTags) { + assertTrue(KnownTagCodec.hasSlot(tagId), Long.toHexString(tagId)); + assertTrue(slots.add(KnownTagCodec.slotOf(tagId)), Long.toHexString(tagId)); + } + } + + @Test + void anOptionalTagOnASpanTypeHasASlot() { + assertTrue(KnownTagCodec.hasSlot(KnownTags.PEER_PORT_ID)); + assertTrue(KnownTagCodec.hasSlot(KnownTags.DB_POOL_NAME_ID)); + } + + @Test + void aTagOnNoSpanTypeHasNoSlot() { + // the tracer overlay's routing-only keys appear on no modeled span type + assertFalse(KnownTagCodec.hasSlot(KnownTags.RESOURCE_NAME_ID)); + } + + @Test + void theSlotDoesNotChangeTheSerialOrFlags() { + long tagId = KnownTags.HTTP_METHOD_ID; + + assertTrue(KnownTagCodec.hasSlot(tagId)); + assertEquals(KnownTags.HTTP_METHOD_NAME, KnownTagCodec.nameOf(tagId)); + assertTrue(KnownTagCodec.isIntercepted(tagId)); + assertFalse(KnownTagCodec.isTraceLevel(tagId)); + } + + @Test + void anUnslottedTagSharesTheReservedTopBit() { + // An occupancy bit of 1L << slotOf(id) needs no branch for NO_SLOT: a long shift uses only the + // low 6 bits of its count, so 0xFFFF lands on bit 63, which no real slot uses + assertEquals(1L << 63, 1L << KnownTagCodec.slotOf(KnownTags.RESOURCE_NAME_ID)); + assertTrue(KnownTagCodec.slotCount() < 63, "real slots must stay below the shared bit 63"); + } +} From b78f5bb609b331b0481b7a1da3811ec67f8df3a6 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 14 Oct 2026 22:36:09 -0400 Subject: [PATCH 2/6] Treat a custom tag's zero id as unslotted hasSlot(0L) decoded slot 0, so the id keyOf gives a custom tag looked like the known tag in slot 0. It now returns false for 0L. The slot-count test also allows the full 63 slots the generator permits. Co-Authored-By: Claude Opus 5.5 --- .../src/main/java/datadog/trace/api/KnownTagCodec.java | 7 +++++-- .../java/datadog/trace/api/KnownTagCodecSlotTest.java | 9 ++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java index 6e09e4d1aad..81465b1b7b4 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -81,9 +81,12 @@ public static int slotOf(long tagId) { return (int) (tagId >>> SLOT_SHIFT) & 0xFFFF; } - /** True if the tag has a slot, below {@link #slotCount()}. */ + /** + * True if the tag has a slot, below {@link #slotCount()}. False for {@code 0L}, the id {@link + * #keyOf} gives a custom tag, which would otherwise decode as slot 0. + */ public static boolean hasSlot(long tagId) { - return slotOf(tagId) != NO_SLOT; + return tagId != 0L && slotOf(tagId) != NO_SLOT; } /** Slots in use: every slotted tag's slot is below this. */ diff --git a/internal-api/src/test/java/datadog/trace/api/KnownTagCodecSlotTest.java b/internal-api/src/test/java/datadog/trace/api/KnownTagCodecSlotTest.java index 4dd09044be8..ded02b397e5 100644 --- a/internal-api/src/test/java/datadog/trace/api/KnownTagCodecSlotTest.java +++ b/internal-api/src/test/java/datadog/trace/api/KnownTagCodecSlotTest.java @@ -67,11 +67,18 @@ void theSlotDoesNotChangeTheSerialOrFlags() { assertFalse(KnownTagCodec.isTraceLevel(tagId)); } + @Test + void aCustomTagHasNoSlot() { + // keyOf gives a custom tag the id 0L, whose slot bits decode as slot 0 + assertEquals(0L, KnownTagCodec.keyOf("my.custom.tag")); + assertFalse(KnownTagCodec.hasSlot(0L)); + } + @Test void anUnslottedTagSharesTheReservedTopBit() { // An occupancy bit of 1L << slotOf(id) needs no branch for NO_SLOT: a long shift uses only the // low 6 bits of its count, so 0xFFFF lands on bit 63, which no real slot uses assertEquals(1L << 63, 1L << KnownTagCodec.slotOf(KnownTags.RESOURCE_NAME_ID)); - assertTrue(KnownTagCodec.slotCount() < 63, "real slots must stay below the shared bit 63"); + assertTrue(KnownTagCodec.slotCount() <= 63, "real slots must stay below the shared bit 63"); } } From 06bac778ee6ff6062bc9a61eaf98b590eef14651 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 14 Oct 2026 22:51:10 -0400 Subject: [PATCH 3/6] Say a slot bit is only a may-be-present signal A span can still hold two tags that share a slot, so slotOf's presence bit is exact only when it is clear. Co-Authored-By: Claude Opus 5.5 --- .../src/main/java/datadog/trace/api/KnownTagCodec.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java index 81465b1b7b4..a405caf8e5b 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -73,9 +73,10 @@ public static boolean isIntercepted(long tagId) { /** * The tag's slot, or {@link #NO_SLOT}. The generator colors tags that appear together on a span - * type with distinct slots, so a span's slotted tags never collide: {@code 1L << slotOf(id)} is - * an exact presence bit for them. Every tag without a slot shares bit 63, which no real slot - * uses. + * type with distinct slots, so when instrumentation follows its span type, {@code 1L << + * slotOf(id)} gives each of a span's tags its own presence bit. A span can still hold two tags + * that share a slot, so a clear bit means the tag is absent, but a set bit only means it may be + * present. Every tag without a slot shares bit 63, which no real slot uses. */ public static int slotOf(long tagId) { return (int) (tagId >>> SLOT_SHIFT) & 0xFFFF; From a50186cd055329625bc6db77c7b7b1441b3778b2 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 15 Oct 2026 17:38:17 -0400 Subject: [PATCH 4/6] Let rarely set tags share slots in the coloring A span type can carry more tags than the occupancy long has slots: a CI Visibility test span carries about 100. Most of those are set only some of the time (retries, ITR, pull request info, ...), and falling back to NO_SLOT would put them all on bit 63, which is then set on most spans and turns every check of those tags into a scan. `slot: shared` marks a tag as rarely set. The coloring keeps a shared tag apart from every tag that isn't shared in its cliques, but lets two shared tags take the same slot. Tags that aren't shared are colored first, so they take the low, exact slots; shared tags fill the slots left free. A span type then fits as long as its always-present tags do. With no tag marked shared, the coloring and every id are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../tagRegistry/KnownTagsEmitter.kt | 1 + .../buildlogic/tagRegistry/TagConventions.kt | 15 +++++++ .../buildlogic/tagRegistry/TagRegistry.kt | 28 +++++++++---- .../tagRegistry/TagRegistryGeneratorTest.kt | 40 +++++++++++++++++++ .../java/datadog/trace/api/KnownTagCodec.java | 7 ++-- 5 files changed, 81 insertions(+), 10 deletions(-) diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt index 63715868802..d450c9bbf73 100644 --- a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt @@ -86,6 +86,7 @@ object KnownTagsEmitter { append(if (t.slot == TagRegistry.NO_SLOT) " + no slot" else " + slot=${t.slot}") if (t.traceLevel) append(" + trace-level") if (t.intercepted) append(" + intercepted") + if (t.sharedSlot) append(" + shared-slot") if (t.otelName != null) append(" -> ${escape(t.otelName)}") for ((direction, otel) in t.scopedOtelNames) append(" -> ${escape(otel)} (${direction.yamlKey})") appendLine(" <${escape(t.required)}>") diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt index 27ca3cfba0f..1354659beee 100644 --- a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt @@ -56,6 +56,12 @@ class TagConventions private constructor( * The flag requires an `otel-name` and cannot be used on a tag declared per direction. */ val spanKindNeutral: Boolean = false, + /** + * `slot: shared`: the tag is rarely set, so it may share a dense-store slot with other shared + * tags. It still gets a slot apart from every tag that isn't shared, so tags that are always + * present keep exact presence bits. + */ + val sharedSlot: Boolean = false, ) { /** The identity's label, as reports show it; see [TagIdentity.label]. */ val name: String @@ -538,6 +544,7 @@ class TagConventions private constructor( required = (m["required"] as? String) ?: "optional", otelName = parseOtelName(m), spanKindNeutral = parseSpanKindNeutral(m), + sharedSlot = parseSlot(m), ) } ?: emptyList() } @@ -549,6 +556,14 @@ class TagConventions private constructor( return raw } + /** Reads `slot`, whose only value is `shared`; absent means the tag gets a slot of its own. */ + private fun parseSlot(m: Map): Boolean { + if (!m.containsKey("slot")) return false + val raw = m["slot"] + require(raw == "shared") { "tag '${m["dd-name"]}' has slot '$raw'; the only value is 'shared'" } + return true + } + private fun parseSpanKindNeutral(m: Map): Boolean { if (!m.containsKey("span-kind-neutral")) return false val raw = m["span-kind-neutral"] diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt index cf1eb3e572b..8e20e5ffa19 100644 --- a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt @@ -11,7 +11,9 @@ package datadog.buildlogic.tagRegistry * resolved tags appear together on one span, so they form a clique and need distinct slots; the * trace-level tier is its own clique (it lives in a separate TagMap). Tags that never share a clique * may share a slot, so the slot count stays bounded by the largest clique, small enough for one - * occupancy `long`. Every tag resolved onto a span type, or trace-level, is slotted whatever its + * occupancy `long`. A tag marked `slot: shared` (rarely set) is kept apart only from the tags in its + * cliques that aren't shared: shared tags may share slots with each other, so a span type with more + * tags than slots fits as long as its always-present tags do. Every tag resolved onto a span type, or trace-level, is slotted whatever its * requirement level; a tag on no modeled span type (the tracer overlay's routing-only keys, tags of * unmodeled span types) gets [NO_SLOT]. Nothing in the tracer reads slots yet: they are for * per-span structures that want an exact, compact presence word, such as a dense tag store or a @@ -41,6 +43,8 @@ class TagRegistry private constructor(val tags: List) { val otelName: String? = null, /** The tag's OpenTelemetry name on spans of each direction that renames it, when [otelName] is null. */ val scopedOtelNames: Map = emptyMap(), + /** Rarely set, so its slot may be shared with other shared tags; see [TagConventions.Tag.sharedSlot]. */ + val sharedSlot: Boolean = false, ) { /** The tag as reports and generated constant names show it. */ val name: String @@ -81,6 +85,9 @@ class TagRegistry private constructor(val tags: List) { require(decls.all { it.type == first.type }) { "'${first.ddName}' is declared per direction with different types: ${decls.map { it.type }}" } + require(decls.all { it.sharedSlot == first.sharedSlot }) { + "'${first.ddName}' is declared per direction with and without slot: shared" + } val serial = FIRST_SERIAL + i val isTraceLevel = first.ddName in traceLevel val isIntercepted = first.ddName in conv.interceptedNames @@ -97,6 +104,7 @@ class TagRegistry private constructor(val tags: List) { id = encode(serial, isTraceLevel, isIntercepted, slot), otelName = otelName, scopedOtelNames = scoped, + sharedSlot = first.sharedSlot, ) } @@ -106,9 +114,10 @@ class TagRegistry private constructor(val tags: List) { /** * Colors the co-occurrence graph of slotted tags: tags resolved onto one concrete span type, or - * both trace-level, must get distinct slots. Greedy, most-constrained first (by clique count, - * then name), so the result is a pure function of the conventions. A name declared per direction - * is one tag, so it is one vertex. + * both trace-level, must get distinct slots -- except that two tags marked `slot: shared` may + * share one. Tags that aren't shared are colored first, most-constrained first (by clique count, + * then name), so they take the low slots; shared tags then fill the slots their cliques' other + * tags leave free. The result is a pure function of the conventions. */ private fun colorSlots(conv: TagConventions): Map { val cliques = ArrayList>() @@ -116,23 +125,28 @@ class TagRegistry private constructor(val tags: List) { cliques.add(conv.resolve(type).map { it.ddName }.toSet()) } cliques.add(conv.traceLevelTags().map { it.ddName }.toSet()) + val shared = conv.allDeclaredTags().filter { it.sharedSlot }.map { it.ddName }.toSet() val neighbors = HashMap>() for (clique in cliques) { for (a in clique) { val adjacent = neighbors.getOrPut(a) { HashSet() } - for (b in clique) if (a != b) adjacent.add(b) + for (b in clique) if (a != b && !(a in shared && b in shared)) adjacent.add(b) } } val cliqueCount = neighbors.keys.associateWith { tag -> cliques.count { tag in it } } - val order = neighbors.keys.sortedWith(compareByDescending { cliqueCount.getValue(it) }.thenBy { it }) + val order = + neighbors.keys.sortedWith( + compareBy { it in shared }.thenByDescending { cliqueCount.getValue(it) }.thenBy { it } + ) val slots = HashMap() for (tag in order) { val taken = neighbors.getValue(tag).mapNotNull { slots[it] }.toSet() var slot = 0 while (slot in taken) slot++ require(slot < MAX_SLOTS) { - "coloring needs more than $MAX_SLOTS slots at '$tag'; one occupancy long holds $MAX_SLOTS" + "coloring needs more than $MAX_SLOTS slots at '$tag'; one occupancy long holds $MAX_SLOTS. " + + "Mark tags set only some of the time `slot: shared`." } slots[tag] = slot } diff --git a/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt index 6db58e6f27b..b1eb89795e6 100644 --- a/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt +++ b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt @@ -599,6 +599,46 @@ class TagRegistryGeneratorTest { ) } + @Test + fun `shared tags may share a slot with each other but not with tags that are always set`() { + val yaml = directory.conventionsFile( + """ + span_types: + test: + span-kind: internal + tags: + - {dd-name: test.name, required: required} + - {dd-name: test.suite, required: required} + - {dd-name: test.is_retry, slot: shared} + - {dd-name: test.is_new, slot: shared} + - {dd-name: test.skip_reason, slot: shared} + """ + ) + val output = File(directory, "generated") + + TagRegistryGenerator.generate(yaml, output) + + // test.name and test.suite take exact slots 0 and 1; the three shared tags all take slot 2. + val knownTags = contents(output).getValue("java/datadog/trace/api/KnownTags.java") + assertThat(knownTags) + .contains( + "TEST_IS_NEW_ID = 0x0001000200000000L", + "TEST_IS_RETRY_ID = 0x0002000200000000L", + "TEST_NAME_ID = 0x0003000000000000L", + "TEST_SKIP_REASON_ID = 0x0004000200000000L", + "TEST_SUITE_ID = 0x0005000100000000L", + "public static final int SLOT_COUNT = 3;", + "+ shared-slot" + ) + } + + @Test + fun `a slot value other than shared is rejected`() { + assertThatIllegalArgumentException() + .isThrownBy { tagConventions("span_types: {base: {tags: [{dd-name: x.y, slot: rare}]}}") } + .withMessageContaining("the only value is 'shared'") + } + private fun tagConventions(@Language("yaml") yamlText: String) = TagConventions.parse( ObjectMapper(YAMLFactory()).readValue( directory.conventionsFile(yamlText), diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java index a405caf8e5b..73d1b68dab5 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -74,9 +74,10 @@ public static boolean isIntercepted(long tagId) { /** * The tag's slot, or {@link #NO_SLOT}. The generator colors tags that appear together on a span * type with distinct slots, so when instrumentation follows its span type, {@code 1L << - * slotOf(id)} gives each of a span's tags its own presence bit. A span can still hold two tags - * that share a slot, so a clear bit means the tag is absent, but a set bit only means it may be - * present. Every tag without a slot shares bit 63, which no real slot uses. + * slotOf(id)} gives each of a span's tags its own presence bit -- except tags marked {@code slot: + * shared}, which are rarely set and may share a slot with each other. A span can still hold two + * tags that share a slot, so a clear bit means the tag is absent, but a set bit only means it may + * be present. Every tag without a slot shares bit 63, which no real slot uses. */ public static int slotOf(long tagId) { return (int) (tagId >>> SLOT_SHIFT) & 0xFFFF; From 866ebc2007c147ac4d9e15e4f2b70408c7117ca3 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 15 Oct 2026 18:13:41 -0400 Subject: [PATCH 5/6] Share slots by requirement level instead of a slot marker Replaces the `slot: shared` marker with the requirement level the conventions already carry, so slots stay out of the registry's vocabulary. A ref can override the level per span type, so a tag can be always present on one span type and occasional on another. Tags that are required, conditional or recommended on some span type are colored first and always get exact slots. Optional and opt_in tags get exact slots too while any are free; only when a span type runs out do two of them share one, on the slot the fewest already hold. Coloring fails only if a span type's non-optional tags alone need more than 63. On the current conventions nothing overflows: SLOT_COUNT stays 24 and no tag shares a slot, though some slots move with the new order. KnownTags marks a tag that shares a slot with `+ shares slot`. Co-Authored-By: Claude Opus 5.5 --- .../tagRegistry/KnownTagsEmitter.kt | 2 +- .../buildlogic/tagRegistry/TagConventions.kt | 15 ---- .../buildlogic/tagRegistry/TagRegistry.kt | 89 +++++++++++-------- .../tagRegistry/TagRegistryGeneratorTest.kt | 51 ++++++----- .../java/datadog/trace/api/KnownTagCodec.java | 4 +- 5 files changed, 87 insertions(+), 74 deletions(-) diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt index d450c9bbf73..5d430f9d7ae 100644 --- a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt @@ -86,7 +86,7 @@ object KnownTagsEmitter { append(if (t.slot == TagRegistry.NO_SLOT) " + no slot" else " + slot=${t.slot}") if (t.traceLevel) append(" + trace-level") if (t.intercepted) append(" + intercepted") - if (t.sharedSlot) append(" + shared-slot") + if (t.sharesSlot) append(" + shares slot") if (t.otelName != null) append(" -> ${escape(t.otelName)}") for ((direction, otel) in t.scopedOtelNames) append(" -> ${escape(otel)} (${direction.yamlKey})") appendLine(" <${escape(t.required)}>") diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt index 1354659beee..27ca3cfba0f 100644 --- a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagConventions.kt @@ -56,12 +56,6 @@ class TagConventions private constructor( * The flag requires an `otel-name` and cannot be used on a tag declared per direction. */ val spanKindNeutral: Boolean = false, - /** - * `slot: shared`: the tag is rarely set, so it may share a dense-store slot with other shared - * tags. It still gets a slot apart from every tag that isn't shared, so tags that are always - * present keep exact presence bits. - */ - val sharedSlot: Boolean = false, ) { /** The identity's label, as reports show it; see [TagIdentity.label]. */ val name: String @@ -544,7 +538,6 @@ class TagConventions private constructor( required = (m["required"] as? String) ?: "optional", otelName = parseOtelName(m), spanKindNeutral = parseSpanKindNeutral(m), - sharedSlot = parseSlot(m), ) } ?: emptyList() } @@ -556,14 +549,6 @@ class TagConventions private constructor( return raw } - /** Reads `slot`, whose only value is `shared`; absent means the tag gets a slot of its own. */ - private fun parseSlot(m: Map): Boolean { - if (!m.containsKey("slot")) return false - val raw = m["slot"] - require(raw == "shared") { "tag '${m["dd-name"]}' has slot '$raw'; the only value is 'shared'" } - return true - } - private fun parseSpanKindNeutral(m: Map): Boolean { if (!m.containsKey("span-kind-neutral")) return false val raw = m["span-kind-neutral"] diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt index 8e20e5ffa19..46af5c20776 100644 --- a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt @@ -11,11 +11,11 @@ package datadog.buildlogic.tagRegistry * resolved tags appear together on one span, so they form a clique and need distinct slots; the * trace-level tier is its own clique (it lives in a separate TagMap). Tags that never share a clique * may share a slot, so the slot count stays bounded by the largest clique, small enough for one - * occupancy `long`. A tag marked `slot: shared` (rarely set) is kept apart only from the tags in its - * cliques that aren't shared: shared tags may share slots with each other, so a span type with more - * tags than slots fits as long as its always-present tags do. Every tag resolved onto a span type, or trace-level, is slotted whatever its - * requirement level; a tag on no modeled span type (the tracer overlay's routing-only keys, tags of - * unmodeled span types) gets [NO_SLOT]. Nothing in the tracer reads slots yet: they are for + * occupancy `long`. A span type with more tags than slots fits as long as its tags at a + * `required`, `conditional` or `recommended` level do: on overflow, its `optional` and `opt_in` tags + * share slots with each other. Every tag resolved onto a span type, or trace-level, is slotted; a + * tag on no modeled span type (the tracer overlay's routing-only keys, tags of unmodeled span types) + * gets [NO_SLOT]. Nothing in the tracer reads slots yet: they are for * per-span structures that want an exact, compact presence word, such as a dense tag store or a * span builder's tag ledger. * @@ -43,8 +43,8 @@ class TagRegistry private constructor(val tags: List) { val otelName: String? = null, /** The tag's OpenTelemetry name on spans of each direction that renames it, when [otelName] is null. */ val scopedOtelNames: Map = emptyMap(), - /** Rarely set, so its slot may be shared with other shared tags; see [TagConventions.Tag.sharedSlot]. */ - val sharedSlot: Boolean = false, + /** The tag's slot is shared with another optional tag on one of its span types, after an overflow. */ + val sharesSlot: Boolean = false, ) { /** The tag as reports and generated constant names show it. */ val name: String @@ -75,7 +75,7 @@ class TagRegistry private constructor(val tags: List) { fun build(conv: TagConventions): TagRegistry { val traceLevel = conv.traceLevelTags().map { it.ddName }.toSet() val renames = conv.otelMappings().groupBy { it.tag.ddName } - val slots = colorSlots(conv) + val (slots, sharing) = colorSlots(conv) // One tag per Datadog name, in a stable order (by name) so serials -- and therefore ids -- are // a pure function of the input. A name's declarations per direction merge into its one tag. @@ -85,9 +85,6 @@ class TagRegistry private constructor(val tags: List) { require(decls.all { it.type == first.type }) { "'${first.ddName}' is declared per direction with different types: ${decls.map { it.type }}" } - require(decls.all { it.sharedSlot == first.sharedSlot }) { - "'${first.ddName}' is declared per direction with and without slot: shared" - } val serial = FIRST_SERIAL + i val isTraceLevel = first.ddName in traceLevel val isIntercepted = first.ddName in conv.interceptedNames @@ -104,7 +101,7 @@ class TagRegistry private constructor(val tags: List) { id = encode(serial, isTraceLevel, isIntercepted, slot), otelName = otelName, scopedOtelNames = scoped, - sharedSlot = first.sharedSlot, + sharesSlot = first.ddName in sharing, ) } @@ -112,45 +109,67 @@ class TagRegistry private constructor(val tags: List) { return TagRegistry(tags) } + /** Requirement levels whose tags may share a slot with each other when a span type overflows. */ + private val OCCASIONAL = setOf("optional", "opt_in") + /** * Colors the co-occurrence graph of slotted tags: tags resolved onto one concrete span type, or - * both trace-level, must get distinct slots -- except that two tags marked `slot: shared` may - * share one. Tags that aren't shared are colored first, most-constrained first (by clique count, - * then name), so they take the low slots; shared tags then fill the slots their cliques' other - * tags leave free. The result is a pure function of the conventions. + * both trace-level, get distinct slots while the slots last. Tags that are `required`, + * `conditional` or `recommended` on some span type are colored first, most-constrained first (by + * clique count, then name), so they take the low slots and always get exact ones. Two tags that + * are both `optional` or `opt_in` on a span type may share a slot there, but only when its other + * slots are taken, and then on the slot the fewest of them already hold. Returns the slots, and + * the tags that share one. The result is a pure function of the conventions. */ - private fun colorSlots(conv: TagConventions): Map { - val cliques = ArrayList>() - for (type in conv.concreteTypes()) { - cliques.add(conv.resolve(type).map { it.ddName }.toSet()) - } - cliques.add(conv.traceLevelTags().map { it.ddName }.toSet()) - val shared = conv.allDeclaredTags().filter { it.sharedSlot }.map { it.ddName }.toSet() + private fun colorSlots(conv: TagConventions): Pair, Set> { + val cliques = ArrayList>() + for (type in conv.concreteTypes()) cliques.add(conv.resolve(type)) + cliques.add(conv.traceLevelTags()) - val neighbors = HashMap>() + // Hard edges keep two tags apart; soft edges, between two occasional tags, are kept while the + // slots last. + val hard = HashMap>() + val soft = HashMap>() + val core = HashSet() for (clique in cliques) { for (a in clique) { - val adjacent = neighbors.getOrPut(a) { HashSet() } - for (b in clique) if (a != b && !(a in shared && b in shared)) adjacent.add(b) + if (a.required !in OCCASIONAL) core.add(a.ddName) + val hardAdjacent = hard.getOrPut(a.ddName) { HashSet() } + val softAdjacent = soft.getOrPut(a.ddName) { HashSet() } + for (b in clique) { + if (a.ddName == b.ddName) continue + if (a.required in OCCASIONAL && b.required in OCCASIONAL) { + softAdjacent.add(b.ddName) + } else { + hardAdjacent.add(b.ddName) + } + } } } - val cliqueCount = neighbors.keys.associateWith { tag -> cliques.count { tag in it } } + for ((tag, adjacent) in soft) adjacent.removeAll(hard.getValue(tag)) + val cliqueCount = hard.keys.associateWith { tag -> cliques.count { c -> c.any { it.ddName == tag } } } val order = - neighbors.keys.sortedWith( - compareBy { it in shared }.thenByDescending { cliqueCount.getValue(it) }.thenBy { it } + hard.keys.sortedWith( + compareBy { it !in core }.thenByDescending { cliqueCount.getValue(it) }.thenBy { it } ) val slots = HashMap() + val sharing = HashSet() for (tag in order) { - val taken = neighbors.getValue(tag).mapNotNull { slots[it] }.toSet() - var slot = 0 - while (slot in taken) slot++ - require(slot < MAX_SLOTS) { + val taken = hard.getValue(tag).mapNotNull { slots[it] }.toSet() + val softTaken = soft.getValue(tag).mapNotNull { slots[it] } + val free = (0 until MAX_SLOTS).filter { it !in taken } + require(free.isNotEmpty()) { "coloring needs more than $MAX_SLOTS slots at '$tag'; one occupancy long holds $MAX_SLOTS. " + - "Mark tags set only some of the time `slot: shared`." + "Only tags that are optional or opt_in on a span type may share slots there." + } + val slot = free.minBy { s -> softTaken.count { it == s } * MAX_SLOTS + s } + if (slot in softTaken) { + sharing.add(tag) + soft.getValue(tag).filter { slots[it] == slot }.forEach { sharing.add(it) } } slots[tag] = slot } - return slots + return slots to sharing } /** diff --git a/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt index b1eb89795e6..928691fd370 100644 --- a/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt +++ b/build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt @@ -600,43 +600,52 @@ class TagRegistryGeneratorTest { } @Test - fun `shared tags may share a slot with each other but not with tags that are always set`() { + fun `optional tags share slots only when a span type overflows`() { + // 60 required tags and 6 optional ones on one span type: 66 tags, over the 63 slots. + val required = (0 until 60).joinToString("\n") { " - {dd-name: t.req$it, required: required}" } + val optional = (0 until 6).joinToString("\n") { " - {dd-name: t.opt$it}" } + val yaml = directory.conventionsFile("span_types:\n test:\n span-kind: internal\n tags:\n$required\n$optional\n") + val output = File(directory, "generated") + + TagRegistryGenerator.generate(yaml, output) + + // The required tags take exact slots 0-59; the optional ones spread over the 3 left, 2 each. + val knownTags = contents(output).getValue("java/datadog/trace/api/KnownTags.java") + val slotOf = Regex("""T_(\w+)_ID = 0x[0-9A-F]{4}([0-9A-F]{4})""").findAll(knownTags) + .associate { it.groupValues[1] to it.groupValues[2].toInt(16) } + assertThat((0 until 60).map { slotOf.getValue("REQ$it") }.toSet()).isEqualTo((0 until 60).toSet()) + assertThat((0 until 6).map { slotOf.getValue("OPT$it") }.groupingBy { it }.eachCount()) + .isEqualTo(mapOf(60 to 2, 61 to 2, 62 to 2)) + assertThat(Regex("shares slot").findAll(knownTags).count()).isEqualTo(6) + assertThat(knownTags).contains("public static final int SLOT_COUNT = 63;") + } + + @Test + fun `optional tags keep exact slots when the span type fits`() { val yaml = directory.conventionsFile( """ span_types: test: span-kind: internal - tags: - - {dd-name: test.name, required: required} - - {dd-name: test.suite, required: required} - - {dd-name: test.is_retry, slot: shared} - - {dd-name: test.is_new, slot: shared} - - {dd-name: test.skip_reason, slot: shared} + tags: [{dd-name: test.name, required: required}, {dd-name: test.is_retry}, {dd-name: test.is_new}] """ ) val output = File(directory, "generated") TagRegistryGenerator.generate(yaml, output) - // test.name and test.suite take exact slots 0 and 1; the three shared tags all take slot 2. val knownTags = contents(output).getValue("java/datadog/trace/api/KnownTags.java") - assertThat(knownTags) - .contains( - "TEST_IS_NEW_ID = 0x0001000200000000L", - "TEST_IS_RETRY_ID = 0x0002000200000000L", - "TEST_NAME_ID = 0x0003000000000000L", - "TEST_SKIP_REASON_ID = 0x0004000200000000L", - "TEST_SUITE_ID = 0x0005000100000000L", - "public static final int SLOT_COUNT = 3;", - "+ shared-slot" - ) + assertThat(knownTags).contains("public static final int SLOT_COUNT = 3;").doesNotContain("shares slot") } @Test - fun `a slot value other than shared is rejected`() { + fun `a span type whose non-optional tags need more than 63 slots is rejected`() { + val tags = (0 until 64).joinToString("\n") { " - {dd-name: t.req$it, required: recommended}" } + val yaml = directory.conventionsFile("span_types:\n test:\n span-kind: internal\n tags:\n$tags\n") + assertThatIllegalArgumentException() - .isThrownBy { tagConventions("span_types: {base: {tags: [{dd-name: x.y, slot: rare}]}}") } - .withMessageContaining("the only value is 'shared'") + .isThrownBy { TagRegistryGenerator.generate(yaml, File(directory, "generated")) } + .withMessageContaining("coloring needs more than 63 slots") } private fun tagConventions(@Language("yaml") yamlText: String) = TagConventions.parse( diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java index 73d1b68dab5..c9a6efb84cf 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -74,8 +74,8 @@ public static boolean isIntercepted(long tagId) { /** * The tag's slot, or {@link #NO_SLOT}. The generator colors tags that appear together on a span * type with distinct slots, so when instrumentation follows its span type, {@code 1L << - * slotOf(id)} gives each of a span's tags its own presence bit -- except tags marked {@code slot: - * shared}, which are rarely set and may share a slot with each other. A span can still hold two + * slotOf(id)} gives each of a span's tags its own presence bit -- except that on a span type with + * more tags than slots, its optional tags share slots with each other. A span can still hold two * tags that share a slot, so a clear bit means the tag is absent, but a set bit only means it may * be present. Every tag without a slot shares bit 63, which no real slot uses. */ From 5ada412c00f5962e6b7dc40c8277f2c5e8b55d91 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 15 Oct 2026 18:19:09 -0400 Subject: [PATCH 6/6] Share slots only on span types that overflow An optional tag pushed onto a shared slot by one span type's overflow could land on the slot of an optional tag of another span type, one that fits. Optional tags now get soft edges only on span types with more tags than slots, so a span type that fits never has two tags on one slot. Co-Authored-By: Claude Opus 5.5 --- .../datadog/buildlogic/tagRegistry/TagRegistry.kt | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt index 46af5c20776..bd6937f6703 100644 --- a/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt +++ b/build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt @@ -116,9 +116,10 @@ class TagRegistry private constructor(val tags: List) { * Colors the co-occurrence graph of slotted tags: tags resolved onto one concrete span type, or * both trace-level, get distinct slots while the slots last. Tags that are `required`, * `conditional` or `recommended` on some span type are colored first, most-constrained first (by - * clique count, then name), so they take the low slots and always get exact ones. Two tags that - * are both `optional` or `opt_in` on a span type may share a slot there, but only when its other - * slots are taken, and then on the slot the fewest of them already hold. Returns the slots, and + * clique count, then name), so they take the low slots and always get exact ones. Only on a span + * type with more tags than slots may two tags that are both `optional` or `opt_in` there share a + * slot, and only once the free slots run out, on the slot the fewest of them already hold; a span + * type that fits never has two tags on one slot. Returns the slots, and * the tags that share one. The result is a pure function of the conventions. */ private fun colorSlots(conv: TagConventions): Pair, Set> { @@ -126,19 +127,20 @@ class TagRegistry private constructor(val tags: List) { for (type in conv.concreteTypes()) cliques.add(conv.resolve(type)) cliques.add(conv.traceLevelTags()) - // Hard edges keep two tags apart; soft edges, between two occasional tags, are kept while the - // slots last. + // Hard edges keep two tags apart; soft edges, between two occasional tags on a span type that + // overflows, are kept while the slots last. val hard = HashMap>() val soft = HashMap>() val core = HashSet() for (clique in cliques) { + val overflows = clique.size > MAX_SLOTS for (a in clique) { if (a.required !in OCCASIONAL) core.add(a.ddName) val hardAdjacent = hard.getOrPut(a.ddName) { HashSet() } val softAdjacent = soft.getOrPut(a.ddName) { HashSet() } for (b in clique) { if (a.ddName == b.ddName) continue - if (a.required in OCCASIONAL && b.required in OCCASIONAL) { + if (overflows && a.required in OCCASIONAL && b.required in OCCASIONAL) { softAdjacent.add(b.ddName) } else { hardAdjacent.add(b.ddName)