Store known span tags densely, behind an experimental flag (phase 3b - dense tag storage)

build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt

@@ -96,6 +96,7 @@ object KnownTagsEmitter {
           appendLine("  public static final String ${otelNameC(t.ddName)} = \"${escape(t.otelName)}\";")
         }
         append("// makeTagId(serial=${t.serial})")
+        append(if (t.slot == TagRegistry.NO_SLOT) " + no slot" else " + slot=${t.slot}")
         if (t.traceLevel) append(" + trace-level")
         if (t.intercepted) append(" + intercepted")
         if (t.scopedOtelNames.isNotEmpty()) append(" + scoped-alias")
@@ -114,6 +115,9 @@ object KnownTagsEmitter {
       // Every known tag's serial is below SERIAL_LIMIT; serial 0 is no tag.
       val serialLimit = (reg.tags.maxOfOrNull { it.serial } ?: 0) + 1
       appendLine("  public static final int SERIAL_LIMIT = $serialLimit;")
+      // Slots in use: every slotted tag's slot is below SLOT_COUNT.
+      val slotCount = (reg.tags.filter { it.slot != TagRegistry.NO_SLOT }.maxOfOrNull { it.slot } ?: -1) + 1
+      appendLine("  public static final int SLOT_COUNT = $slotCount;")
 
       // OpenTelemetry name -> canonical tag name. Validation ensures aliases are distinct from all
       // canonical names. Sort by OTel name to keep output deterministic.

build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt

@@ -2,12 +2,19 @@ package datadog.buildlogic.tagRegistry
 
 /**
  * Assigns tag ids from a parsed [TagConventions]. The id encoding mirrors KnownTagCodec: [63-48
- * serial][47-32 reserved][31-0 flags].
+ * serial][47-32 slot][31-0 flags].
  *
- * <p>An id is a globally unique serial plus two classification bits: trace-level and intercepted. It
- * carries no storage-layout coordinate -- bits [47-32] are held vacant for the co-occurrence slot
- * that the dense tag store assigns by graph coloring, which lands with the dense store itself.
- * Nothing here needs to know how (or whether) a tag is stored.
+ * <p>An id is a globally unique serial, a slot, and two classification bits: trace-level and
+ * intercepted.
+ *
+ * <p>The slot comes from GRAPH COLORING the tag co-occurrence graph. Each concrete span type's
+ * resolved tags appear together on one span, so they form a clique and need distinct slots; the
+ * trace-level tier is its own clique (it lives in a separate TagMap). Tags that never share a clique
+ * may share a slot, so the slot count stays bounded by the largest clique, small enough for one
+ * occupancy `long`. Every tag resolved onto a span type, or trace-level, is slotted whatever its
+ * requirement level; a tag on no modeled span type (the tracer overlay's routing-only keys, tags of
+ * unmodeled span types) gets [NO_SLOT]. Slots are for per-span structures that want an exact,
+ * compact presence word: the dense tag store (DenseTagMap), and in time a span builder's tag ledger.
  *
  * <p>The intercepted bit marks a tag the tracer may route on the set-path (to a span field or a
  * sampling directive instead of tag storage), as listed by the tracer overlay. A setter called with
@@ -26,6 +33,8 @@ class TagRegistry private constructor(val tags: List<Tag>) {
     val serial: Int,
     val traceLevel: Boolean,
     val intercepted: Boolean,
+    /** The tag's slot, or [NO_SLOT] for a tag on no modeled span type. */
+    val slot: Int,
     val id: Long,
     /** The tag's OpenTelemetry name in every direction; null when not renamed, or renamed per direction. */
     val otelName: String? = null,
@@ -43,24 +52,35 @@ class TagRegistry private constructor(val tags: List<Tag>) {
     const val INTERCEPTED = 1L shl 1 // low-32 carve bit 1; mirrors KnownTagCodec.INTERCEPTED
     const val LEVEL_TRACE = 1L shl 2 // low-32 carve bit 2; mirrors KnownTagCodec.LEVEL_TRACE
 
+    const val SLOT_SHIFT = 32 // slot window [47-32]; mirrors KnownTagCodec.SLOT_SHIFT
+    const val NO_SLOT = 0xFFFF // all-ones slot: no modeled span type; mirrors KnownTagCodec.NO_SLOT
+    const val MAX_SLOTS = 63 // fits one occupancy long; bit 63 is left for unslotted tags to share
+
     /**
      * Serials stay below 2^15, so bit 63 of an id is never set: every id is positive, clear of 0
      * (unknown) and of KnownTagCodec's negative lookup sentinels.
      */
     const val SERIAL_CAPACITY = 1 shl 15
 
     /**
-     * Mirrors KnownTagCodec.makeTagId(serial) + intercepted() + traceLevel() -- must stay in sync.
+     * Mirrors KnownTagCodec's layout -- must stay in sync: serial at [63-48], slot at [47-32],
      * HAS_SCOPED_ALIAS at bit 0 (the tag has an OpenTelemetry name in some directions only),
-     * INTERCEPTED at bit 1, LEVEL_TRACE at bit 2, other low bits and the reserved [47-32] window
-     * zero. Every id is made here, and the range check makes each one positive by construction.
+     * INTERCEPTED at bit 1, LEVEL_TRACE at bit 2, other low bits zero. Every id is made here, and
+     * the range check makes each one positive by construction.
      */
-    fun encode(serial: Int, traceLevel: Boolean, intercepted: Boolean, hasScopedAlias: Boolean = false): Long {
+    fun encode(
+      serial: Int,
+      traceLevel: Boolean,
+      intercepted: Boolean,
+      hasScopedAlias: Boolean = false,
+      slot: Int = NO_SLOT,
+    ): Long {
+      require(slot == NO_SLOT || slot in 0 until MAX_SLOTS) { "slot $slot is outside [0, $MAX_SLOTS)" }
       require(serial in FIRST_SERIAL until SERIAL_CAPACITY) {
         "serial $serial is outside [$FIRST_SERIAL, $SERIAL_CAPACITY): more tags would make tag ids " +
           "negative, where KnownTagCodec's lookup sentinels live"
       }
-      var id = serial.toLong() shl 48
+      var id = (serial.toLong() shl 48) or (slot.toLong() shl SLOT_SHIFT)
       if (traceLevel) id = id or LEVEL_TRACE
       if (intercepted) id = id or INTERCEPTED
       if (hasScopedAlias) id = id or HAS_SCOPED_ALIAS
@@ -70,6 +90,7 @@ class TagRegistry private constructor(val tags: List<Tag>) {
     fun build(conv: TagConventions): TagRegistry {
       val traceLevel = conv.traceLevelTags().map { it.ddName }.toSet()
       val renames = conv.otelMappings().groupBy { it.tag.ddName }
+      val slots = colorSlots(conv)
 
       // One tag per Datadog name, in a stable order (by name) so serials -- and therefore ids -- are
       // a pure function of the input. A name's declarations per direction merge into its one tag.
@@ -83,14 +104,16 @@ class TagRegistry private constructor(val tags: List<Tag>) {
           val isTraceLevel = first.ddName in traceLevel
           val isIntercepted = first.ddName in conv.interceptedNames
           val (otelName, scoped) = otelNamesOf(first.ddName, renames[first.ddName].orEmpty())
+          val slot = slots[first.ddName] ?: NO_SLOT
           Tag(
             first.ddName,
             first.type,
             strongestRequirement(decls.map { it.required }),
             serial,
             isTraceLevel,
             isIntercepted,
-            id = encode(serial, isTraceLevel, isIntercepted, scoped.isNotEmpty()),
+            slot,
+            id = encode(serial, isTraceLevel, isIntercepted, scoped.isNotEmpty(), slot),
             otelName = otelName,
             scopedOtelNames = scoped,
           )
@@ -100,6 +123,41 @@ class TagRegistry private constructor(val tags: List<Tag>) {
       return TagRegistry(tags)
     }
 
+    /**
+     * Colors the co-occurrence graph of slotted tags: tags resolved onto one concrete span type, or
+     * both trace-level, must get distinct slots. Greedy, most-constrained first (by clique count,
+     * then name), so the result is a pure function of the conventions. A name declared per direction
+     * is one tag, so it is one vertex.
+     */
+    private fun colorSlots(conv: TagConventions): Map<String, Int> {
+      val cliques = ArrayList<Set<String>>()
+      for (type in conv.concreteTypes()) {
+        cliques.add(conv.resolve(type).map { it.ddName }.toSet())
+      }
+      cliques.add(conv.traceLevelTags().map { it.ddName }.toSet())
+
+      val neighbors = HashMap<String, MutableSet<String>>()
+      for (clique in cliques) {
+        for (a in clique) {
+          val adjacent = neighbors.getOrPut(a) { HashSet() }
+          for (b in clique) if (a != b) adjacent.add(b)
+        }
+      }
+      val cliqueCount = neighbors.keys.associateWith { tag -> cliques.count { tag in it } }
+      val order = neighbors.keys.sortedWith(compareByDescending<String> { cliqueCount.getValue(it) }.thenBy { it })
+      val slots = HashMap<String, Int>()
+      for (tag in order) {
+        val taken = neighbors.getValue(tag).mapNotNull { slots[it] }.toSet()
+        var slot = 0
+        while (slot in taken) slot++
+        require(slot < MAX_SLOTS) {
+          "coloring needs more than $MAX_SLOTS slots at '$tag'; one occupancy long holds $MAX_SLOTS"
+        }
+        slots[tag] = slot
+      }
+      return slots
+    }
+
     /**
      * A tag's renames, as one name for every direction or one name per direction. The rename of a
      * declaration made per direction applies in that declaration's direction.

build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt

@@ -70,16 +70,17 @@ object TagRegistryGenerator {
       """
       # Tag id assignment.  tags=${reg.tags.size}
 
-      # TAGS     serial lvl int id                 required     name
+      # TAGS     serial lvl int slot id                 required     name
       """.trimIndent()
     )
     for (t in reg.tags) {
       appendLine(
-        "  %6d   %s   %s   %-18s %-12s %s".format(
+        "  %6d   %s   %s   %4s %-18s %-12s %s".format(
           Locale.ROOT,
           t.serial,
           if (t.traceLevel) "T" else "-",
           if (t.intercepted) "I" else "-",
+          if (t.slot == TagRegistry.NO_SLOT) "-" else t.slot.toString(),
           "0x%016X".format(Locale.ROOT, t.id),
           t.required,
           t.name

build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt

@@ -558,9 +558,9 @@ class TagRegistryGeneratorTest {
     val knownTags = contents(output).getValue("java/datadog/trace/api/KnownTags.java")
     assertThat(knownTags)
       .contains(
-        "COMPONENT_ID = 0x0001000000000000L",
-        "RESOURCE_NAME_ID = 0x0002000000000002L",
-        "SERVICE_ID = 0x0003000000000002L",
+        "COMPONENT_ID = 0x0001FFFF00000000L",
+        "RESOURCE_NAME_ID = 0x0002FFFF00000002L",
+        "SERVICE_ID = 0x0003FFFF00000002L",
         "public static final int SERVICE_SERIAL_NUM = 3;",
         "public static final int SERIAL_LIMIT = 4;"
       )
@@ -584,6 +584,44 @@ class TagRegistryGeneratorTest {
       .withMessageContaining(message)
   }
 
+  @Test
+  fun `tags on one span type get distinct slots and tags that never co-occur share one`() {
+    val yaml = directory.conventionsFile(
+      """
+      trace_level:
+        tags: [{dd-name: env, required: required}]
+      span_types:
+        base:
+          abstract: true
+          tags: [{dd-name: service, required: required}]
+        http.server:
+          extends: base
+          span-kind: server
+          tags: [{dd-name: http.route, required: required}]
+        db.client:
+          extends: base
+          span-kind: client
+          tags: [{dd-name: db.type, required: required}, {dd-name: db.user}]
+      """
+    )
+    val output = File(directory, "generated")
+
+    TagRegistryGenerator.generate(yaml, output)
+
+    // service is on both span types, so it takes slot 0; db.type and db.user share db.client, so
+    // they take distinct slots, and http.route -- never on a span type with either -- reuses one of
+    // them; env is trace-level (its own TagMap) and reuses slot 0. Every level is slotted.
+    assertThat(contents(output).getValue("java/datadog/trace/api/KnownTags.java"))
+      .contains(
+        "DB_TYPE_ID = 0x0001000100000000L",
+        "DB_USER_ID = 0x0002000200000000L",
+        "ENV_ID = 0x0003000000000004L",
+        "HTTP_ROUTE_ID = 0x0004000100000000L",
+        "SERVICE_ID = 0x0005000000000000L",
+        "public static final int SLOT_COUNT = 3;"
+      )
+  }
+
   private fun tagConventions(@Language("yaml") yamlText: String) = TagConventions.parse(
     ObjectMapper(YAMLFactory()).readValue(
       directory.conventionsFile(yamlText),

dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java

@@ -190,5 +190,11 @@ public final class TracerConfig {
   public static final String TRACE_ORG_GUARD_STRICT = "trace.org.guard.strict";
   public static final String TRACE_ORG_GUARD_TRUSTED_OPMS = "trace.org.guard.trusted.opms";
 
+  /**
+   * Stores known tags that have a dense-store slot in TagMap's dense store (no per-tag entry)
+   * instead of its hash buckets. Experimental, off by default.
+   */
+  public static final String TRACE_DENSE_TAGS_ENABLED = "trace.experimental.dense.tags.enabled";
+
   private TracerConfig() {}
 }

dd-trace-core/src/jmh/java/datadog/trace/core/SetTagBenchmark.java

@@ -125,6 +125,15 @@ public Object knownIntTag_byId() {
     return span.setTag(KnownTags.HTTP_RESEND_COUNT_ID, 2);
   }
 
+  /**
+   * {@link #knownIntTag_byId} with a value outside {@code Integer}'s cache, so a box, if the path
+   * makes one, is a real allocation -- as for most status codes and ports.
+   */
+  @Benchmark
+  public Object knownIntTag_byId_uncached() {
+    return span.setTag(KnownTags.HTTP_RESEND_COUNT_ID, 5432);
+  }
+
   @Benchmark
   public Object knownIntTag_byName() {
     return span.setTag(KnownTags.HTTP_RESEND_COUNT_NAME, 2);