Color known tags into co-occurrence slots in their ids (phase 3a - groundwork - dense tag storage)

build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/KnownTagsEmitter.kt

@@ -83,8 +83,10 @@ object KnownTagsEmitter {
           appendLine("  public static final String ${otelNameC(t.ddName)} = \"${escape(t.otelName)}\";")
         }
         append("// makeTagId(serial=${t.serial})")
+        append(if (t.slot == TagRegistry.NO_SLOT) " + no slot" else " + slot=${t.slot}")
         if (t.traceLevel) append(" + trace-level")
         if (t.intercepted) append(" + intercepted")
+        if (t.sharesSlot) append(" + shares slot")
         if (t.otelName != null) append(" -> ${escape(t.otelName)}")
         for ((direction, otel) in t.scopedOtelNames) append(" -> ${escape(otel)} (${direction.yamlKey})")
         appendLine("  <${escape(t.required)}>")
@@ -100,6 +102,9 @@ object KnownTagsEmitter {
       // Every known tag's serial is below SERIAL_LIMIT; serial 0 is no tag.
       val serialLimit = (reg.tags.maxOfOrNull { it.serial } ?: 0) + 1
       appendLine("  public static final int SERIAL_LIMIT = $serialLimit;")
+      // Slots in use: every slotted tag's slot is below SLOT_COUNT.
+      val slotCount = (reg.tags.filter { it.slot != TagRegistry.NO_SLOT }.maxOfOrNull { it.slot } ?: -1) + 1
+      appendLine("  public static final int SLOT_COUNT = $slotCount;")
 
       // OpenTelemetry name -> canonical tag name. Validation ensures aliases are distinct from all
       // canonical names. Sort by OTel name to keep output deterministic.

build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistry.kt

@@ -2,12 +2,22 @@ package datadog.buildlogic.tagRegistry
 
 /**
  * Assigns tag ids from a parsed [TagConventions]. The id encoding mirrors KnownTagCodec: [63-48
- * serial][47-32 reserved][31-0 flags].
+ * serial][47-32 slot][31-0 flags].
  *
- * <p>An id is a globally unique serial plus two classification bits: trace-level and intercepted. It
- * carries no storage-layout coordinate -- bits [47-32] are held vacant for the co-occurrence slot
- * that the dense tag store assigns by graph coloring, which lands with the dense store itself.
- * Nothing here needs to know how (or whether) a tag is stored.
+ * <p>An id is a globally unique serial, a slot, and two classification bits: trace-level and
+ * intercepted.
+ *
+ * <p>The slot comes from GRAPH COLORING the tag co-occurrence graph. Each concrete span type's
+ * resolved tags appear together on one span, so they form a clique and need distinct slots; the
+ * trace-level tier is its own clique (it lives in a separate TagMap). Tags that never share a clique
+ * may share a slot, so the slot count stays bounded by the largest clique, small enough for one
+ * occupancy `long`. A span type with more tags than slots fits as long as its tags at a
+ * `required`, `conditional` or `recommended` level do: on overflow, its `optional` and `opt_in` tags
+ * share slots with each other. Every tag resolved onto a span type, or trace-level, is slotted; a
+ * tag on no modeled span type (the tracer overlay's routing-only keys, tags of unmodeled span types)
+ * gets [NO_SLOT]. Nothing in the tracer reads slots yet: they are for
+ * per-span structures that want an exact, compact presence word, such as a dense tag store or a
+ * span builder's tag ledger.
  *
  * <p>The intercepted bit marks a tag the tracer may route on the set-path (to a span field or a
  * sampling directive instead of tag storage), as listed by the tracer overlay. A setter called with
@@ -26,11 +36,15 @@ class TagRegistry private constructor(val tags: List<Tag>) {
     val serial: Int,
     val traceLevel: Boolean,
     val intercepted: Boolean,
+    /** The tag's slot, or [NO_SLOT] for a tag on no modeled span type. */
+    val slot: Int,
     val id: Long,
     /** The tag's OpenTelemetry name in every direction; null when not renamed, or renamed per direction. */
     val otelName: String? = null,
     /** The tag's OpenTelemetry name on spans of each direction that renames it, when [otelName] is null. */
     val scopedOtelNames: Map<TagConventions.Direction, String> = emptyMap(),
+    /** The tag's slot is shared with another optional tag on one of its span types, after an overflow. */
+    val sharesSlot: Boolean = false,
   ) {
     /** The tag as reports and generated constant names show it. */
     val name: String
@@ -42,13 +56,17 @@ class TagRegistry private constructor(val tags: List<Tag>) {
     const val INTERCEPTED = 1L shl 1 // low-32 carve bit 1; mirrors KnownTagCodec.INTERCEPTED
     const val LEVEL_TRACE = 1L shl 2 // low-32 carve bit 2; mirrors KnownTagCodec.LEVEL_TRACE
 
+    const val SLOT_SHIFT = 32 // slot window [47-32]; mirrors KnownTagCodec.SLOT_SHIFT
+    const val NO_SLOT = 0xFFFF // all-ones slot: no modeled span type; mirrors KnownTagCodec.NO_SLOT
+    const val MAX_SLOTS = 63 // fits one occupancy long; bit 63 is left for unslotted tags to share
+
     /**
-     * Mirrors KnownTagCodec.makeTagId(serial) + intercepted() + traceLevel() -- must stay in sync.
-     * INTERCEPTED at bit 1, LEVEL_TRACE at bit 2, other low bits and the reserved [47-32] window
-     * zero.
+     * Mirrors KnownTagCodec's layout -- must stay in sync: serial at [63-48], slot at [47-32],
+     * INTERCEPTED at bit 1, LEVEL_TRACE at bit 2, other low bits zero.
      */
-    fun encode(serial: Int, traceLevel: Boolean, intercepted: Boolean): Long {
-      var id = serial.toLong() shl 48
+    fun encode(serial: Int, traceLevel: Boolean, intercepted: Boolean, slot: Int = NO_SLOT): Long {
+      require(slot == NO_SLOT || slot in 0 until MAX_SLOTS) { "slot $slot is outside [0, $MAX_SLOTS)" }
+      var id = (serial.toLong() shl 48) or (slot.toLong() shl SLOT_SHIFT)
       if (traceLevel) id = id or LEVEL_TRACE
       if (intercepted) id = id or INTERCEPTED
       return id
@@ -57,6 +75,7 @@ class TagRegistry private constructor(val tags: List<Tag>) {
     fun build(conv: TagConventions): TagRegistry {
       val traceLevel = conv.traceLevelTags().map { it.ddName }.toSet()
       val renames = conv.otelMappings().groupBy { it.tag.ddName }
+      val (slots, sharing) = colorSlots(conv)
 
       // One tag per Datadog name, in a stable order (by name) so serials -- and therefore ids -- are
       // a pure function of the input. A name's declarations per direction merge into its one tag.
@@ -70,23 +89,91 @@ class TagRegistry private constructor(val tags: List<Tag>) {
           val isTraceLevel = first.ddName in traceLevel
           val isIntercepted = first.ddName in conv.interceptedNames
           val (otelName, scoped) = otelNamesOf(first.ddName, renames[first.ddName].orEmpty())
+          val slot = slots[first.ddName] ?: NO_SLOT
           Tag(
             first.ddName,
             first.type,
             strongestRequirement(decls.map { it.required }),
             serial,
             isTraceLevel,
             isIntercepted,
-            id = encode(serial, isTraceLevel, isIntercepted),
+            slot,
+            id = encode(serial, isTraceLevel, isIntercepted, slot),
             otelName = otelName,
             scopedOtelNames = scoped,
+            sharesSlot = first.ddName in sharing,
           )
         }
 
       validateOtelNames(tags)
       return TagRegistry(tags)
     }
 
+    /** Requirement levels whose tags may share a slot with each other when a span type overflows. */
+    private val OCCASIONAL = setOf("optional", "opt_in")
+
+    /**
+     * Colors the co-occurrence graph of slotted tags: tags resolved onto one concrete span type, or
+     * both trace-level, get distinct slots while the slots last. Tags that are `required`,
+     * `conditional` or `recommended` on some span type are colored first, most-constrained first (by
+     * clique count, then name), so they take the low slots and always get exact ones. Only on a span
+     * type with more tags than slots may two tags that are both `optional` or `opt_in` there share a
+     * slot, and only once the free slots run out, on the slot the fewest of them already hold; a span
+     * type that fits never has two tags on one slot. Returns the slots, and
+     * the tags that share one. The result is a pure function of the conventions.
+     */
+    private fun colorSlots(conv: TagConventions): Pair<Map<String, Int>, Set<String>> {
+      val cliques = ArrayList<List<TagConventions.Tag>>()
+      for (type in conv.concreteTypes()) cliques.add(conv.resolve(type))
+      cliques.add(conv.traceLevelTags())
+
+      // Hard edges keep two tags apart; soft edges, between two occasional tags on a span type that
+      // overflows, are kept while the slots last.
+      val hard = HashMap<String, MutableSet<String>>()
+      val soft = HashMap<String, MutableSet<String>>()
+      val core = HashSet<String>()
+      for (clique in cliques) {
+        val overflows = clique.size > MAX_SLOTS
+        for (a in clique) {
+          if (a.required !in OCCASIONAL) core.add(a.ddName)
+          val hardAdjacent = hard.getOrPut(a.ddName) { HashSet() }
+          val softAdjacent = soft.getOrPut(a.ddName) { HashSet() }
+          for (b in clique) {
+            if (a.ddName == b.ddName) continue
+            if (overflows && a.required in OCCASIONAL && b.required in OCCASIONAL) {
+              softAdjacent.add(b.ddName)
+            } else {
+              hardAdjacent.add(b.ddName)
+            }
+          }
+        }
+      }
+      for ((tag, adjacent) in soft) adjacent.removeAll(hard.getValue(tag))
+      val cliqueCount = hard.keys.associateWith { tag -> cliques.count { c -> c.any { it.ddName == tag } } }
+      val order =
+        hard.keys.sortedWith(
+          compareBy<String> { it !in core }.thenByDescending { cliqueCount.getValue(it) }.thenBy { it }
+        )
+      val slots = HashMap<String, Int>()
+      val sharing = HashSet<String>()
+      for (tag in order) {
+        val taken = hard.getValue(tag).mapNotNull { slots[it] }.toSet()
+        val softTaken = soft.getValue(tag).mapNotNull { slots[it] }
+        val free = (0 until MAX_SLOTS).filter { it !in taken }
+        require(free.isNotEmpty()) {
+          "coloring needs more than $MAX_SLOTS slots at '$tag'; one occupancy long holds $MAX_SLOTS. " +
+            "Only tags that are optional or opt_in on a span type may share slots there."
+        }
+        val slot = free.minBy { s -> softTaken.count { it == s } * MAX_SLOTS + s }
+        if (slot in softTaken) {
+          sharing.add(tag)
+          soft.getValue(tag).filter { slots[it] == slot }.forEach { sharing.add(it) }
+        }
+        slots[tag] = slot
+      }
+      return slots to sharing
+    }
+
     /**
      * A tag's renames, as one name for every direction or one name per direction. The rename of a
      * declaration made per direction applies in that declaration's direction.

build-logic/tag-registry/src/main/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGenerator.kt

@@ -70,16 +70,17 @@ object TagRegistryGenerator {
       """
       # Tag id assignment.  tags=${reg.tags.size}
 
-      # TAGS     serial lvl int id                 required     name
+      # TAGS     serial lvl int slot id                 required     name
       """.trimIndent()
     )
     for (t in reg.tags) {
       appendLine(
-        "  %6d   %s   %s   %-18s %-12s %s".format(
+        "  %6d   %s   %s   %4s %-18s %-12s %s".format(
           Locale.ROOT,
           t.serial,
           if (t.traceLevel) "T" else "-",
           if (t.intercepted) "I" else "-",
+          if (t.slot == TagRegistry.NO_SLOT) "-" else t.slot.toString(),
           "0x%016X".format(Locale.ROOT, t.id),
           t.required,
           t.name

build-logic/tag-registry/src/test/kotlin/datadog/buildlogic/tagRegistry/TagRegistryGeneratorTest.kt

@@ -535,9 +535,9 @@ class TagRegistryGeneratorTest {
     val knownTags = contents(output).getValue("java/datadog/trace/api/KnownTags.java")
     assertThat(knownTags)
       .contains(
-        "COMPONENT_ID = 0x0001000000000000L",
-        "RESOURCE_NAME_ID = 0x0002000000000002L",
-        "SERVICE_ID = 0x0003000000000002L",
+        "COMPONENT_ID = 0x0001FFFF00000000L",
+        "RESOURCE_NAME_ID = 0x0002FFFF00000002L",
+        "SERVICE_ID = 0x0003FFFF00000002L",
         "public static final int SERVICE_SERIAL_NUM = 3;",
         "public static final int SERIAL_LIMIT = 4;"
       )
@@ -561,6 +561,93 @@ class TagRegistryGeneratorTest {
       .withMessageContaining(message)
   }
 
+  @Test
+  fun `tags on one span type get distinct slots and tags that never co-occur share one`() {
+    val yaml = directory.conventionsFile(
+      """
+      trace_level:
+        tags: [{dd-name: env, required: required}]
+      span_types:
+        base:
+          abstract: true
+          tags: [{dd-name: service, required: required}]
+        http.server:
+          extends: base
+          span-kind: server
+          tags: [{dd-name: http.route, required: required}]
+        db.client:
+          extends: base
+          span-kind: client
+          tags: [{dd-name: db.type, required: required}, {dd-name: db.user}]
+      """
+    )
+    val output = File(directory, "generated")
+
+    TagRegistryGenerator.generate(yaml, output)
+
+    // service is on both span types, so it takes slot 0; db.type and db.user share db.client, so
+    // they take distinct slots, and http.route -- never on a span type with either -- reuses one of
+    // them; env is trace-level (its own TagMap) and reuses slot 0. Every level is slotted.
+    assertThat(contents(output).getValue("java/datadog/trace/api/KnownTags.java"))
+      .contains(
+        "DB_TYPE_ID = 0x0001000100000000L",
+        "DB_USER_ID = 0x0002000200000000L",
+        "ENV_ID = 0x0003000000000004L",
+        "HTTP_ROUTE_ID = 0x0004000100000000L",
+        "SERVICE_ID = 0x0005000000000000L",
+        "public static final int SLOT_COUNT = 3;"
+      )
+  }
+
+  @Test
+  fun `optional tags share slots only when a span type overflows`() {
+    // 60 required tags and 6 optional ones on one span type: 66 tags, over the 63 slots.
+    val required = (0 until 60).joinToString("\n") { "        - {dd-name: t.req$it, required: required}" }
+    val optional = (0 until 6).joinToString("\n") { "        - {dd-name: t.opt$it}" }
+    val yaml = directory.conventionsFile("span_types:\n  test:\n    span-kind: internal\n    tags:\n$required\n$optional\n")
+    val output = File(directory, "generated")
+
+    TagRegistryGenerator.generate(yaml, output)
+
+    // The required tags take exact slots 0-59; the optional ones spread over the 3 left, 2 each.
+    val knownTags = contents(output).getValue("java/datadog/trace/api/KnownTags.java")
+    val slotOf = Regex("""T_(\w+)_ID = 0x[0-9A-F]{4}([0-9A-F]{4})""").findAll(knownTags)
+      .associate { it.groupValues[1] to it.groupValues[2].toInt(16) }
+    assertThat((0 until 60).map { slotOf.getValue("REQ$it") }.toSet()).isEqualTo((0 until 60).toSet())
+    assertThat((0 until 6).map { slotOf.getValue("OPT$it") }.groupingBy { it }.eachCount())
+      .isEqualTo(mapOf(60 to 2, 61 to 2, 62 to 2))
+    assertThat(Regex("shares slot").findAll(knownTags).count()).isEqualTo(6)
+    assertThat(knownTags).contains("public static final int SLOT_COUNT = 63;")
+  }
+
+  @Test
+  fun `optional tags keep exact slots when the span type fits`() {
+    val yaml = directory.conventionsFile(
+      """
+      span_types:
+        test:
+          span-kind: internal
+          tags: [{dd-name: test.name, required: required}, {dd-name: test.is_retry}, {dd-name: test.is_new}]
+      """
+    )
+    val output = File(directory, "generated")
+
+    TagRegistryGenerator.generate(yaml, output)
+
+    val knownTags = contents(output).getValue("java/datadog/trace/api/KnownTags.java")
+    assertThat(knownTags).contains("public static final int SLOT_COUNT = 3;").doesNotContain("shares slot")
+  }
+
+  @Test
+  fun `a span type whose non-optional tags need more than 63 slots is rejected`() {
+    val tags = (0 until 64).joinToString("\n") { "        - {dd-name: t.req$it, required: recommended}" }
+    val yaml = directory.conventionsFile("span_types:\n  test:\n    span-kind: internal\n    tags:\n$tags\n")
+
+    assertThatIllegalArgumentException()
+      .isThrownBy { TagRegistryGenerator.generate(yaml, File(directory, "generated")) }
+      .withMessageContaining("coloring needs more than 63 slots")
+  }
+
   private fun tagConventions(@Language("yaml") yamlText: String) = TagConventions.parse(
     ObjectMapper(YAMLFactory()).readValue(
       directory.conventionsFile(yamlText),

internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java

@@ -13,11 +13,11 @@
  */
 public final class KnownTagCodec {
   /*
-   * tagId bit layout: [63-48 serialNum (16 bits)] [47-32 reserved, zero] [31-0 flags]. serialNum is
+   * tagId bit layout: [63-48 serialNum (16 bits)] [47-32 slot] [31-0 flags]. serialNum is
    * globally unique per known tag and is the whole of the tag's identity — nameOf/
    * openTelemetryNameOf switch on it, and the generator emits each id as a literal. Bits [47-32]
-   * are RESERVED and always zero here: they are the window the dense tag store uses for its
-   * co-occurrence slot coordinate, which arrives with that store. Of the low 32 flag bits, bit 2 is
+   * hold the tag's slot ({@link #slotOf}), assigned by graph coloring so tags on one span type never
+   * share a slot, or {@link #NO_SLOT} for a tag on no modeled span type. Of the low 32 flag bits, bit 2 is
    * the trace/span LEVEL bit (set ⟹ trace-level), and bit 1 is the INTERCEPTED bit. Unknown
    * (string-only) custom tags are NOT known ids — {@code keyOf} returns 0 for them.
    *
@@ -65,16 +65,47 @@ public static boolean isIntercepted(long tagId) {
     return (tagId & INTERCEPTED) != 0L;
   }
 
+  /** Bit position of the slot: bits [47-32] of an id. */
+  public static final int SLOT_SHIFT = 32;
+
+  /** The slot of a known tag on no modeled span type. */
+  public static final int NO_SLOT = 0xFFFF;
+
+  /**
+   * The tag's slot, or {@link #NO_SLOT}. The generator colors tags that appear together on a span
+   * type with distinct slots, so when instrumentation follows its span type, {@code 1L <<
+   * slotOf(id)} gives each of a span's tags its own presence bit -- except that on a span type with
+   * more tags than slots, its optional tags share slots with each other. A span can still hold two
+   * tags that share a slot, so a clear bit means the tag is absent, but a set bit only means it may
+   * be present. Every tag without a slot shares bit 63, which no real slot uses.
+   */
+  public static int slotOf(long tagId) {
+    return (int) (tagId >>> SLOT_SHIFT) & 0xFFFF;
+  }
+
+  /**
+   * True if the tag has a slot, below {@link #slotCount()}. False for {@code 0L}, the id {@link
+   * #keyOf} gives a custom tag, which would otherwise decode as slot 0.
+   */
+  public static boolean hasSlot(long tagId) {
+    return tagId != 0L && slotOf(tagId) != NO_SLOT;
+  }
+
+  /** Slots in use: every slotted tag's slot is below this. */
+  public static int slotCount() {
+    return KnownTags.SLOT_COUNT;
+  }
+
   /** Returns the tagId with the {@link #LEVEL_TRACE} flag set. */
   public static long traceLevel(long tagId) {
     return tagId | LEVEL_TRACE;
   }
 
   /**
-   * Builds a tagId from its {@code serialNum} (globally unique per known tag). The reserved [47-32]
-   * window and the low 32 bits are zero, so the id is fully determined by the serial — the
-   * generator emits it as a literal. Inverse of {@link #serialNum}. Intended for the code generator
-   * and tests.
+   * Builds an id carrying only {@code serialNum}: slot 0 and no flags, so it equals a tag's real id
+   * only when that tag has neither. Enough for serial-keyed lookups such as {@link #nameOf};
+   * compare against a {@code KnownTags} constant, never against this. Inverse of {@link
+   * #serialNum}. Intended for tests and serial-keyed lookups.
    */
   public static long makeTagId(int serialNum) {
     return (long) serialNum << 48;