From 98b34e36ac073c400e6ab4aa1bf52a1f34044143 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 14:40:56 -0400 Subject: [PATCH 01/52] Add EmbeddingSupport.contains(hashes, names, name) helper Mirrors StringIndex#contains for the static-arrays path, so callers of the embedded/parallel-array form don't need to spell out indexOf(...) >= 0 themselves. Co-Authored-By: Claude Sonnet 5 --- .../src/main/java/datadog/trace/util/StringIndex.java | 5 +++++ .../test/java/datadog/trace/util/StringIndexTest.java | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/internal-api/src/main/java/datadog/trace/util/StringIndex.java b/internal-api/src/main/java/datadog/trace/util/StringIndex.java index 868558e0c38..1358c56f425 100644 --- a/internal-api/src/main/java/datadog/trace/util/StringIndex.java +++ b/internal-api/src/main/java/datadog/trace/util/StringIndex.java @@ -334,6 +334,11 @@ public static int indexOf(int[] hashes, String[] names, String name) { return indexOf(hashes, names, name, hash(name)); } + /** {@code indexOf(hashes, names, name) >= 0}. Mirrors {@link StringIndex#contains}. */ + public static boolean contains(int[] hashes, String[] names, String name) { + return indexOf(hashes, names, name) >= 0; + } + /** Number of slots — the length to size parallel payload arrays to. */ public static int numSlots(int[] hashes) { return hashes.length; diff --git a/internal-api/src/test/java/datadog/trace/util/StringIndexTest.java b/internal-api/src/test/java/datadog/trace/util/StringIndexTest.java index 46fa68d040e..966d0d0d4ae 100644 --- a/internal-api/src/test/java/datadog/trace/util/StringIndexTest.java +++ b/internal-api/src/test/java/datadog/trace/util/StringIndexTest.java @@ -81,6 +81,16 @@ void support_create_then_indexOf() { assertEquals(-1, EmbeddingSupport.indexOf(d.hashes, d.names, "q")); } + @Test + void support_contains_internedAndCopy_andMiss() { + Data d = EmbeddingSupport.create("foo", "bar", "baz"); + + assertTrue( + EmbeddingSupport.contains(d.hashes, d.names, "foo")); // interned literal -> == fast path + assertTrue(EmbeddingSupport.contains(d.hashes, d.names, new String("bar"))); // non-interned + assertFalse(EmbeddingSupport.contains(d.hashes, d.names, "nope")); + } + /** Controlled hashes force collision, linear-probe wraparound, and the already-present path. */ @Test void put_and_indexOf_collisionAndWraparound() { From a44015139c0a18176b1e50a15bc10e353adfedfb Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 14:41:14 -0400 Subject: [PATCH 02/52] Add hitFresh scenario and hash-dispatch pollution to ImmutableSetBenchmark Reusing the same interned/cached-hash key instances for both building a structure and measuring hit lookups against it understates real hit cost -- String#equals's == fast path and String#hashCode's cached result pay nothing extra for a key already touched during setUp. The new hitFresh scenario looks up separate, never-touched String instances instead, alongside the existing hit (interned literal) and miss (already representative) scenarios. Also pulls in BenchmarkUtils#polluteHashDispatch so the shared hashCode/equals call sites are already megamorphic before the StringSet arms measure their own lookups, matching production where those call sites are hit by every hash-based structure in the JVM. Replaced the javadoc's stale, partial-run tables/findings with a full hit/hitFresh/miss re-run across all six structures together (Fork(5), Threads(8)) -- confirms hitFresh is the slowest of the three scenarios for every hash-based structure, not merely slower than hit as previously guessed from partial data. Co-Authored-By: Claude Sonnet 5 --- .../datadog/trace/util/BenchmarkUtils.java | 59 ++++++++ .../trace/util/ImmutableSetBenchmark.java | 126 ++++++++++++++---- 2 files changed, 162 insertions(+), 23 deletions(-) create mode 100644 internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java new file mode 100644 index 00000000000..56ae1c3ebc1 --- /dev/null +++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java @@ -0,0 +1,59 @@ +package datadog.trace.util; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** Shared setup helpers for JMH benchmarks in this module. */ +public final class BenchmarkUtils { + private BenchmarkUtils() {} + + private static final Object[] DEFAULT_DECOY_KEYS = { + "decoy", 1, 1L, 1.0d, Boolean.TRUE, new Object() + }; + + /** + * Exercises {@link HashSet}/{@link java.util.HashMap} and the tracer's {@link + * CollectionUtils#tryMakeImmutableSet} immutable sets with several distinct key classes, so their + * internal {@code hashCode()}/{@code equals()} dispatch -- a call site shared JVM-wide by every + * instance of that structure in the process, regardless of which specific instance or call site + * invokes {@code add}/{@code contains} -- is already megamorphic before a benchmark measures + * lookups against a single key type. + * + *

This matches production: those shared internal call sites are hit by every hash-based + * structure in the JVM across whatever key types the whole application uses, so they're + * realistically almost always megamorphic. An isolated benchmark that only ever looks up one key + * type (e.g. {@code String}) would otherwise leave them artificially monomorphic for the entire + * run, understating real dispatch cost. + * + *

Deliberately does not touch the benchmark's own {@code contains}/{@code add} call sites -- + * those are realistically free to specialize per caller, the way a genuinely hot, narrowly-typed + * call site would in production. + * + *

Not to be confused with the CHA-defeat decoys in {@code SingleThreadedMapBenchmark}/{@code + * ThreadSafeMapBenchmark} ({@code KeyStrategy} implementors referenced only so they're loaded, + * never invoked): that technique denies class-hierarchy analysis a single-implementor bet for a + * narrow, dd-trace-java-owned interface, and works by class-loading alone. It doesn't apply here + * -- {@code Object.hashCode()}/{@code equals()} already have countless implementors loaded in any + * real JVM, so a single-implementor CHA bet was never available for them. What gates their + * dispatch is the interpreter's per-call-site type profile, which only invocation can pollute -- + * hence this helper actually calls {@code add}/{@code contains}, rather than just loading + * classes. + */ + public static void polluteHashDispatch() { + polluteHashDispatch(DEFAULT_DECOY_KEYS); + } + + public static void polluteHashDispatch(Object... decoyKeys) { + HashSet scratchHashSet = new HashSet<>(); + for (Object key : decoyKeys) { + scratchHashSet.add(key); + scratchHashSet.contains(key); + } + + Set scratchImmutableSet = CollectionUtils.tryMakeImmutableSet(Arrays.asList(decoyKeys)); + for (Object key : decoyKeys) { + scratchImmutableSet.contains(key); + } + } +} diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java index f0604e6ddd4..66d3c5feb6c 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java @@ -44,37 +44,69 @@ * indirection cost of the wrapper. * * - *

Lookups are interned (the {@code ==} fast path where a structure has one); misses are short - * and never present. + *

Hit lookups come in two flavors, because reusing the exact same key instances for both + * building a structure and measuring lookups against it is its own validity bug, independent of + * hash-dispatch pollution: {@link String#equals} takes an {@code ==} fast path, and {@link + * String#hashCode()} caches its result in a field on first call, so a key instance that was already + * inserted (or previously looked up) pays neither real cost again. * - *

JDK 17 results (Apple M1, quiet machine, {@code @Fork(5)}, {@code @Threads(8)}; M ops/s = - * millions): + *

    + *
  • {@code hit} -- looks up the same interned {@link #STRINGS} literals used to build every + * structure. This is realistic and worth keeping (fixed header/config-key lookups commonly + * are interned literals), but identity with the stored element is inherent to interning, not + * a choice this benchmark makes -- two equal literals are always the same instance. + *
  • {@code hitFresh} -- looks up {@link #FRESH_STRINGS}, separate {@code new String(..)} + * instances never touched by {@link #setUp}, so each carries an uncached hash and forces a + * real {@code equals()} beyond the {@code ==} check. Models keys arriving from + * parsing/concatenation/deserialization rather than literals. Only meaningful for the + * hash-based structures ({@code hashSet}, {@code tracerImmutableSet}, {@code stringIndex}, + * {@code stringIndex_embedded}); not measured for {@code array}/{@code sortedArray}/{@code + * treeSet}. + *
+ * + *

Misses are already representative of both effects for free: {@link #MISSES} is built via + * concatenation (never interned) and never touched by {@link #setUp}. + * + *

Full re-run, all six structures across {@code hit}/{@code hitFresh}/{@code miss} together, + * with {@link BenchmarkUtils#polluteHashDispatch()} in effect (Apple M1, Java 8u382 -- the + * repo-default {@code jmh} test launcher, no {@code -PtestJvm} override --, {@code @Fork(5)}, + * {@code @Threads(8)}; M ops/s = millions): * * 

{@code
- * Structure                           hit    miss
- * stringIndex_embedded (static)      2320    2159    (fastest)
- * hashSet                            2198    2134
- * stringIndex (inst)                 2098  1548 *    (* miss bimodal -- see caveat)
- * tracerImmutableSet                 1914    1663    (Set.copyOf / SetN)
- * array                               941     589
- * sortedArray                         685     610
- * treeSet                             657     610
+ * Structure                    hit   hitFresh    miss
+ * stringIndex_embedded (static) 2098      1563    2030    (fastest hit and miss)
+ * hashSet                       1723      1276    1823
+ * stringIndex (inst)            1883      1184 *  1700 *  (* bimodal -- see caveat)
+ * tracerImmutableSet            1632      1232    1625    (Set.copyOf / SetN)
+ * array                          854         -     495
+ * sortedArray                    713         -     613
+ * treeSet                        646         -     544
  * }
* *

Key findings: * *

    - *
  • The static {@code EmbeddingSupport} path is the fastest — it beats {@code HashSet} on hit - * and miss and crushes the scan/search/tree forms. + *
  • The static {@code EmbeddingSupport} path is the fastest on both hit and miss -- it beats + * {@code HashSet} on both and crushes the scan/search/tree forms. *
  • {@code stringIndex} (the instance wrapper) trails {@code EmbeddingSupport} by the - * field-load indirection (~10% on hit), landing near {@code HashSet} — fine off the hot path, - * prefer {@code EmbeddingSupport} on it. - *
  • {@link java.util.Set#copyOf} ({@code SetN}, the agent's compact fixed-set form) is ~1.2x - * behind {@code EmbeddingSupport} on hit but the most compact (~27% smaller — no - * cached hashes, no 2x table). So StringIndex's edge over {@code SetN} is speed + the {@code - * indexOf}->parallel-array capability, not footprint; over {@code HashSet} it wins both. - *
  • {@code array} / {@code sortedArray} / {@code treeSet} trail the hashed structures, most on + * field-load indirection, but on this run actually leads {@code HashSet} on hit; miss is + * noisy (see bimodal caveat below) and not a reliable comparison point. Prefer {@code + * EmbeddingSupport} on the hot path regardless. + *
  • {@link java.util.Set#copyOf} ({@code SetN}, the agent's compact fixed-set form) trails + * {@code EmbeddingSupport} on every scenario but remains the most compact (~27% + * smaller -- no cached hashes, no 2x table). So StringIndex's edge over {@code SetN} is speed + * + the {@code indexOf}->parallel-array capability, not footprint. + *
  • {@code array} / {@code sortedArray} / {@code treeSet} trail every hashed structure, most on * miss. + *
  • {@code hitFresh} is the slowest of the three scenarios for every hash-based + * structure -- clearly below both {@code hit} and {@code miss}, not merely below {@code hit} + * as previously guessed. This makes sense once the two failure shapes are compared: a miss + * usually short-circuits on the first hash mismatch during probing and rarely reaches {@code + * equals()}, while a {@code hitFresh} lookup must probe until it finds the match and pay a + * real, uncached {@code equals()} there -- so it is not simply "the honest version of hit", + * it exercises a genuinely more expensive path than either {@code hit} (cached hash + {@code + * ==}) or {@code miss} (hash-only rejection). Superseded an earlier partial-data guess that + * {@code hitFresh} would land at the same cost as {@code miss}. *
* *

Caveat — the instance {@code stringIndex} miss is bimodal across forks (confirmed at @@ -100,6 +132,21 @@ public class ImmutableSetBenchmark { /** Distinct String instances that are never present, for the miss path. */ static final String[] MISSES = newMisses(); + /** + * Equal-content, non-interned, never-before-hashed copies of {@link #STRINGS}, built once here + * and never touched by {@link #setUp} -- so a lookup against them can't ride the {@code ==} fast + * path or a hash cached during set construction. See {@code hitFresh} in the class javadoc. + */ + static final String[] FRESH_STRINGS = newFreshStrings(); + + static String[] newFreshStrings() { + String[] fresh = new String[STRINGS.length]; + for (int i = 0; i < STRINGS.length; ++i) { + fresh[i] = new String(STRINGS[i]); + } + return fresh; + } + static String[] newMisses() { String[] misses = new String[STRINGS.length * 4]; for (int i = 0; i < misses.length; ++i) { @@ -131,6 +178,8 @@ static String[] newMisses() { @Setup(Level.Trial) public void setUp() { + BenchmarkUtils.polluteHashDispatch(); + array = STRINGS; sortedArray = Arrays.copyOf(STRINGS, STRINGS.length); Arrays.sort(sortedArray); @@ -144,6 +193,7 @@ public void setUp() { @State(Scope.Thread) public static class Cursor { int hitIndex = 0; + int hitFreshIndex = 0; int missIndex = 0; String nextHit() { @@ -155,6 +205,16 @@ String nextHit() { return STRINGS[i]; } + /** See {@code hitFresh} in the class javadoc. */ + String nextHitFresh() { + int i = hitFreshIndex + 1; + if (i >= FRESH_STRINGS.length) { + i = 0; + } + hitFreshIndex = i; + return FRESH_STRINGS[i]; + } + String nextMiss() { int i = missIndex + 1; if (i >= MISSES.length) { @@ -199,6 +259,11 @@ public boolean hashSet_hit(Cursor cursor) { return hashSet.contains(cursor.nextHit()); } + @Benchmark + public boolean hashSet_hitFresh(Cursor cursor) { + return hashSet.contains(cursor.nextHitFresh()); + } + @Benchmark public boolean hashSet_miss(Cursor cursor) { return hashSet.contains(cursor.nextMiss()); @@ -219,6 +284,11 @@ public boolean tracerImmutableSet_hit(Cursor cursor) { return tracerImmutableSet.contains(cursor.nextHit()); } + @Benchmark + public boolean tracerImmutableSet_hitFresh(Cursor cursor) { + return tracerImmutableSet.contains(cursor.nextHitFresh()); + } + @Benchmark public boolean tracerImmutableSet_miss(Cursor cursor) { return tracerImmutableSet.contains(cursor.nextMiss()); @@ -229,6 +299,11 @@ public boolean stringIndex_hit(Cursor cursor) { return stringIndex.contains(cursor.nextHit()); } + @Benchmark + public boolean stringIndex_hitFresh(Cursor cursor) { + return stringIndex.contains(cursor.nextHitFresh()); + } + @Benchmark public boolean stringIndex_miss(Cursor cursor) { return stringIndex.contains(cursor.nextMiss()); @@ -236,11 +311,16 @@ public boolean stringIndex_miss(Cursor cursor) { @Benchmark public boolean stringIndex_embedded_hit(Cursor cursor) { - return StringIndex.EmbeddingSupport.indexOf(SI_HASHES, SI_NAMES, cursor.nextHit()) >= 0; + return StringIndex.EmbeddingSupport.contains(SI_HASHES, SI_NAMES, cursor.nextHit()); + } + + @Benchmark + public boolean stringIndex_embedded_hitFresh(Cursor cursor) { + return StringIndex.EmbeddingSupport.contains(SI_HASHES, SI_NAMES, cursor.nextHitFresh()); } @Benchmark public boolean stringIndex_embedded_miss(Cursor cursor) { - return StringIndex.EmbeddingSupport.indexOf(SI_HASHES, SI_NAMES, cursor.nextMiss()) >= 0; + return StringIndex.EmbeddingSupport.contains(SI_HASHES, SI_NAMES, cursor.nextMiss()); } } From ae7207a0b0d9456d081c6481f91e7e963d1545a8 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 20:10:42 -0400 Subject: [PATCH 03/52] Restructure StringIndex put/indexOf around a single induction variable Recompute the slot index each iteration as (h + probes) & mask instead of carrying a separately-incremented cursor, so the loop has one canonical induction variable for the JIT to reason about. --- .../src/main/java/datadog/trace/util/StringIndex.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/StringIndex.java b/internal-api/src/main/java/datadog/trace/util/StringIndex.java index 1358c56f425..66644e0fecf 100644 --- a/internal-api/src/main/java/datadog/trace/util/StringIndex.java +++ b/internal-api/src/main/java/datadog/trace/util/StringIndex.java @@ -74,7 +74,7 @@ public int indexOf(String name) { } public boolean contains(String name) { - return indexOf(name) >= 0; + return EmbeddingSupport.indexOf(this.hashes, this.names, name) >= 0; } /** Table size — allocate parallel payload arrays of this length. */ @@ -288,8 +288,8 @@ public static long[] mapLongValues(String[] names, ToLongFunction fn) { */ static int put(int[] hashes, String[] names, String name, int h) { final int mask = hashes.length - 1; - int i = h & mask; - for (int probes = 0; probes <= mask; probes++, i = (i + 1) & mask) { + for (int probes = 0; probes <= mask; probes++) { + int i = (h + probes) & mask; if (hashes[i] == 0) { hashes[i] = h; names[i] = name; @@ -313,8 +313,8 @@ static int put(int[] hashes, String[] names, String name, int h) { */ public static int indexOf(int[] hashes, String[] names, String name, int h) { final int mask = hashes.length - 1; - int i = h & mask; - for (int probes = 0; probes <= mask; probes++, i = (i + 1) & mask) { + for (int probes = 0; probes <= mask; probes++) { + int i = (h + probes) & mask; int sh = hashes[i]; if (sh == 0) { return -1; From 69e55098fcb7eeeca98372b216824dc2b51a2020 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 20:10:52 -0400 Subject: [PATCH 04/52] Split BenchmarkUtils type-profile pollution into add-driving and contains-only helpers populateTypeProfile() drives only contains(), so it works against both mutable and immutable collections (including the collection instance under test itself); populateTypeProfileMutable() keeps the old add()+contains() behavior for callers that need add() dispatch polluted too. --- .../datadog/trace/util/BenchmarkUtils.java | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java index 56ae1c3ebc1..06cd5324b86 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java +++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java @@ -1,8 +1,8 @@ package datadog.trace.util; import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; -import java.util.Set; /** Shared setup helpers for JMH benchmarks in this module. */ public final class BenchmarkUtils { @@ -45,15 +45,35 @@ public static void polluteHashDispatch() { } public static void polluteHashDispatch(Object... decoyKeys) { - HashSet scratchHashSet = new HashSet<>(); + populateTypeProfileMutable(new HashSet<>(), decoyKeys); + populateTypeProfile(CollectionUtils.tryMakeImmutableSet(Arrays.asList(decoyKeys)), decoyKeys); + } + + /** + * The entry point most benchmarks should reach for: pass the same kind of collection instance + * under test (or an equivalent scratch instance). Works for both mutable and immutable + * collections since it only drives {@code contains()} -- the operation every {@link + * java.util.Set} supports, and the one these lookup benchmarks actually measure. + */ + public static void populateTypeProfile(Collection populated) { + populateTypeProfile(populated, DEFAULT_DECOY_KEYS); + } + + public static void populateTypeProfile(Collection populated, Object... decoyKeys) { for (Object key : decoyKeys) { - scratchHashSet.add(key); - scratchHashSet.contains(key); + populated.contains(key); } + } - Set scratchImmutableSet = CollectionUtils.tryMakeImmutableSet(Arrays.asList(decoyKeys)); + /** + * Lower-level control: also drives {@code add()} dispatch, so {@code scratch} must genuinely + * support mutation, and lets the caller pick the decoy keys. Reach for this only when {@code + * add()} dispatch matters too, or the default decoys aren't the right shape. + */ + public static void populateTypeProfileMutable(Collection scratch, Object... decoyKeys) { for (Object key : decoyKeys) { - scratchImmutableSet.contains(key); + scratch.add(key); + scratch.contains(key); } } } From cccabcd2140c4bd489c829190d33421a83b9ac20 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 20:11:00 -0400 Subject: [PATCH 05/52] Scope StringIndex-as-Set guidance to hit-dominated access patterns The instance wrapper reliably beats HashSet on hit (its actual design case) but not on miss/hitFresh, where both are bimodal across forks and the mean already sits at or below HashSet's steady figure. Point miss/fresh-key-heavy callers at EmbeddingSupport directly instead of assuming the wrapper is strictly better than a plain Set. --- .../datadog/trace/util/ImmutableSetBenchmark.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java index 66d3c5feb6c..ce5e7dc5ee2 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java @@ -89,9 +89,15 @@ *
  • The static {@code EmbeddingSupport} path is the fastest on both hit and miss -- it beats * {@code HashSet} on both and crushes the scan/search/tree forms. *
  • {@code stringIndex} (the instance wrapper) trails {@code EmbeddingSupport} by the - * field-load indirection, but on this run actually leads {@code HashSet} on hit; miss is - * noisy (see bimodal caveat below) and not a reliable comparison point. Prefer {@code - * EmbeddingSupport} on the hot path regardless. + * field-load indirection. It reliably beats {@code HashSet} on {@code hit} (its actual design + * case: repeated lookups of a known, fixed name set). On {@code miss} and {@code hitFresh} it + * is not a reliable win -- both are bimodal across forks (see caveat below) and the + * mean in each case already sits at or below {@code HashSet}'s steady figure. For miss- or + * fresh-key-heavy membership use, prefer {@link StringIndex.EmbeddingSupport} directly rather + * than assuming the wrapper is strictly better than a plain {@code Set}. This doesn't apply + * to {@code StringIndex}'s parallel-value (map) use case ({@code mapValues}/{@code lookup}) + * -- that win comes from avoiding boxing and node overhead entirely and is unaffected by any + * of this. *
  • {@link java.util.Set#copyOf} ({@code SetN}, the agent's compact fixed-set form) trails * {@code EmbeddingSupport} on every scenario but remains the most compact (~27% * smaller -- no cached hashes, no 2x table). So StringIndex's edge over {@code SetN} is speed From 1efce439de894c0941ccb5237d188e4dfaa19363 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 20:11:09 -0400 Subject: [PATCH 06/52] Pollute type dispatch in ImmutableMapBenchmark and refresh results Object.hashCode()/equals() are JVM-wide shared call sites hit by every hash-based structure in the process; leaving them monomorphic for a single-key-type run understates real dispatch cost, same reasoning as ImmutableSetBenchmark. Rerun and update the javadoc results table with pollution in effect -- StringIndex's get win over HashMap/TagMap/MapN holds up unconditionally here, unlike the access-pattern-dependent Set case. --- .../trace/util/ImmutableMapBenchmark.java | 64 +++++++++++-------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java index 39aaf82183c..5826e92e10c 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java @@ -44,40 +44,50 @@ * {@code *_sameKey} variants reuse the original interned key instances to show the identity fast * path — which is the common tracer case, since map keys are typically interned tag-name constants. * - *

    JDK 17 results (Apple M1, quiet machine, {@code @Fork(5)}, {@code @Threads(8)}; M ops/s). - * {@code get} uses distinct keys (exercises {@code equals()}); {@code sameKey} reuses the interned - * key (the {@code ==} fast path — the common tracer case): + *

    {@link BenchmarkUtils#polluteHashDispatch()} runs in {@link #setUp}, for the same reason as + * {@link ImmutableSetBenchmark}: {@code Object.hashCode()}/{@code equals()} are JVM-wide shared + * call sites, hit by every hash-based structure in the process (this benchmark's {@code HashMap}, + * {@code LinkedHashMap}, and {@code Map.copyOf}/{@code MapN} all dispatch through them for their + * {@code String} keys) — realistically almost always megamorphic, so leaving them monomorphic for + * the whole run would understate real dispatch cost. * - * 

    {@code
- * Structure                           get sameKey
- * stringIndex_embedded (static)      1498    2081    (fastest)
- * stringIndex (inst)                 1363    1900
- * hashMap                            1216    1850
- * linkedHashMap                      1214       -
- * tagMap                             1167    1386
- * tracerImmutableMap                 1049    1364    (MapN)
- * treeMap                             656       -
- * }
    - * - *

    {@code iterate} (full traversal): + *

    JDK 8 results (Apple M1, quiet machine, {@code @Fork(5)}, {@code @Threads(8)}, with {@link + * BenchmarkUtils#polluteHashDispatch()} in effect; M ops/s). {@code get} uses distinct keys + * (exercises {@code equals()}); {@code sameKey} reuses the interned key (the {@code ==} fast path — + * the common tracer case): * * 

    {@code
- * tagMap.forEach        148    (fastest)
- * linkedHashMap         136
- * tracerImmutableMap    135    (MapN)
- * treeMap               134
- * hashMap               104
- * tagMap (iterator)      96
+ * Structure                get sameKey iterate iterate_forEach
+ * hashMap                 1202    1438     120        -
+ * linkedHashMap           1097       -     127        -
+ * treeMap                  487       -     121        -
+ * tagMap                  1005    1235     109       121
+ * tracerImmutableMap      1052    1249     123        -   (MapN)
+ * stringIndex             1366    1724       -        -
+ * stringIndex_embedded    1479    1846       -        -   (fastest get)
  * }
    * *

    Key findings: * *

      - *
    • StringIndex-as-map ({@code EmbeddingSupport}) is the fastest {@code get} — beating {@code - * HashMap} and {@code Map.copyOf}/{@code MapN}, most on the interned path; the instance - * wrapper trails it by ~10%. (vs {@code MapN} the edge is speed + the slot/parallel-array - * capability, not footprint — see {@link ImmutableSetBenchmark}.) - *
    • {@code TagMap.forEach} (148) beats its own {@code iterator} (96) by ~1.5x: TagMap's + *
    • StringIndex-as-map is a reliable {@code get} win, and it holds up under type-profile + * pollution: {@code stringIndex_embedded} (the {@code static final}-array form) and {@code + * stringIndex} (the instance wrapper) both beat every {@code Map} here by a wide margin, on + * both the {@code equals()} and identity-fast-path lookups. Unlike {@link + * ImmutableSetBenchmark}'s {@code hitFresh} case, there's no bimodality here — this win is + * unconditional, not access-pattern-dependent. + *
    • This table still compares boxed {@code Integer} values ({@code hashMap}/{@code + * linkedHashMap}/{@code treeMap}/{@code tracerImmutableMap} all return {@code Integer}; + * {@code tagMap}/{@code stringIndex} happen to expose primitive {@code int} accessors, but + * that's not exercised as a differentiator here). StringIndex's edge should widen further + * against a {@code Map} once the comparison is against genuinely autoboxed + * reads on both sides — not yet measured. + *
    • {@code treeMap}'s {@code get} has a wide error bar (±129, one fork's measurement iterations + * dropped to ~230-300, the rest sit around 490-610) — a known {@code TreeMap} comparison + * characteristic (uses {@code compareTo} not {@code hashCode}/{@code equals}, so it's + * unaffected by dispatch pollution), not the same warmup-race bimodality investigated in + * {@link ImmutableSetBenchmark}. + *
    • {@code TagMap.forEach} (121) beats its own {@code iterator} (109) by ~10%: TagMap's * structure makes a faithful external {@code Iterator} expensive (externalized cursor + * skip-empty + per-call re-entry + the iterator allocation) — all of which internal {@code * forEach} avoids. Traverse TagMap via {@code forEach}, never its iterator; that gap only @@ -145,6 +155,8 @@ static void fill(Map map) { @Setup(Level.Trial) public void setUp() { + BenchmarkUtils.polluteHashDispatch(); + hashMap = new HashMap<>(); fill(hashMap); linkedHashMap = new LinkedHashMap<>(); From 786bd6875d54fd6ef32d8ca2e5cf5f67a835bed0 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 1 Sep 2026 16:20:09 -0400 Subject: [PATCH 07/52] Tighten benchmark javadoc wording per review Apply the reviewer-suggested rewordings for BenchmarkUtils, ImmutableSetBenchmark, ImmutableMapBenchmark, and StringIndex: trim narration and restate a couple of claims (fresh-string hash caching, contains() contract) more precisely. Co-Authored-By: Claude Sonnet 5 --- .../datadog/trace/util/BenchmarkUtils.java | 48 ++++------- .../trace/util/ImmutableMapBenchmark.java | 52 +++++------- .../trace/util/ImmutableSetBenchmark.java | 80 +++++-------------- .../java/datadog/trace/util/StringIndex.java | 6 +- 4 files changed, 58 insertions(+), 128 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java index 06cd5324b86..4aa5ea7556a 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java +++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java @@ -13,32 +13,14 @@ private BenchmarkUtils() {} }; /** - * Exercises {@link HashSet}/{@link java.util.HashMap} and the tracer's {@link - * CollectionUtils#tryMakeImmutableSet} immutable sets with several distinct key classes, so their - * internal {@code hashCode()}/{@code equals()} dispatch -- a call site shared JVM-wide by every - * instance of that structure in the process, regardless of which specific instance or call site - * invokes {@code add}/{@code contains} -- is already megamorphic before a benchmark measures - * lookups against a single key type. + * Makes the internal {@code hashCode()} and {@code equals()} call sites of common hash-based + * collections megamorphic before measurement. * - *

      This matches production: those shared internal call sites are hit by every hash-based - * structure in the JVM across whatever key types the whole application uses, so they're - * realistically almost always megamorphic. An isolated benchmark that only ever looks up one key - * type (e.g. {@code String}) would otherwise leave them artificially monomorphic for the entire - * run, understating real dispatch cost. - * - *

      Deliberately does not touch the benchmark's own {@code contains}/{@code add} call sites -- - * those are realistically free to specialize per caller, the way a genuinely hot, narrowly-typed - * call site would in production. - * - *

      Not to be confused with the CHA-defeat decoys in {@code SingleThreadedMapBenchmark}/{@code - * ThreadSafeMapBenchmark} ({@code KeyStrategy} implementors referenced only so they're loaded, - * never invoked): that technique denies class-hierarchy analysis a single-implementor bet for a - * narrow, dd-trace-java-owned interface, and works by class-loading alone. It doesn't apply here - * -- {@code Object.hashCode()}/{@code equals()} already have countless implementors loaded in any - * real JVM, so a single-implementor CHA bet was never available for them. What gates their - * dispatch is the interpreter's per-call-site type profile, which only invocation can pollute -- - * hence this helper actually calls {@code add}/{@code contains}, rather than just loading - * classes. + *

      HotSpot records receiver classes at each virtual call site. A benchmark using only {@code + * String} keys leaves the sites inside these shared implementations monomorphic, allowing C2 to + * devirtualize and inline them. Production uses many key types, so the same sites are often + * megamorphic and retain virtual dispatch. Exercising several key classes avoids reporting + * unrealistically fast collection lookups. */ public static void polluteHashDispatch() { polluteHashDispatch(DEFAULT_DECOY_KEYS); @@ -50,10 +32,12 @@ public static void polluteHashDispatch(Object... decoyKeys) { } /** - * The entry point most benchmarks should reach for: pass the same kind of collection instance - * under test (or an equivalent scratch instance). Works for both mutable and immutable - * collections since it only drives {@code contains()} -- the operation every {@link - * java.util.Set} supports, and the one these lookup benchmarks actually measure. + * Exercises {@code contains()} with the default decoy keys. + * + *

      Use the collection implementation under test; the instance itself may be a scratch object. + * HotSpot stores receiver-type profiles at bytecode call sites, so every instance executing that + * implementation contributes to the same internal {@code hashCode()} and {@code equals()} + * profiles. The collection may be mutable or immutable. */ public static void populateTypeProfile(Collection populated) { populateTypeProfile(populated, DEFAULT_DECOY_KEYS); @@ -65,11 +49,7 @@ public static void populateTypeProfile(Collection populated, Object... d } } - /** - * Lower-level control: also drives {@code add()} dispatch, so {@code scratch} must genuinely - * support mutation, and lets the caller pick the decoy keys. Reach for this only when {@code - * add()} dispatch matters too, or the default decoys aren't the right shape. - */ + /** Exercises {@code add()} and {@code contains()} on a mutable collection. */ public static void populateTypeProfileMutable(Collection scratch, Object... decoyKeys) { for (Object key : decoyKeys) { scratch.add(key); diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java index 5826e92e10c..7f21e08656f 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java @@ -44,17 +44,16 @@ * {@code *_sameKey} variants reuse the original interned key instances to show the identity fast * path — which is the common tracer case, since map keys are typically interned tag-name constants. * - *

      {@link BenchmarkUtils#polluteHashDispatch()} runs in {@link #setUp}, for the same reason as - * {@link ImmutableSetBenchmark}: {@code Object.hashCode()}/{@code equals()} are JVM-wide shared - * call sites, hit by every hash-based structure in the process (this benchmark's {@code HashMap}, - * {@code LinkedHashMap}, and {@code Map.copyOf}/{@code MapN} all dispatch through them for their - * {@code String} keys) — realistically almost always megamorphic, so leaving them monomorphic for - * the whole run would understate real dispatch cost. + *

      {@link BenchmarkUtils#polluteHashDispatch()} prepares shared {@code hashCode()} and {@code + * equals()} call sites before measurement. This avoids giving hash-based structures an + * unrealistically monomorphic type profile. * - *

      JDK 8 results (Apple M1, quiet machine, {@code @Fork(5)}, {@code @Threads(8)}, with {@link - * BenchmarkUtils#polluteHashDispatch()} in effect; M ops/s). {@code get} uses distinct keys - * (exercises {@code equals()}); {@code sameKey} reuses the interned key (the {@code ==} fast path — - * the common tracer case): + *

      A virtual call site is monomorphic when it has observed one receiver class, + * polymorphic when it has observed a small set, and megamorphic when no small, stable + * set dominates. HotSpot can usually devirtualize and inline the monomorphic case, sometimes a + * small polymorphic one; megamorphic sites generally retain virtual dispatch. + * + *

      Results on an Apple M1 with Java 8u382, {@code @Fork(5)}, and {@code @Threads(8)} (M ops/s): * * 

      {@code
  * Structure                get sameKey iterate iterate_forEach
@@ -64,34 +63,19 @@
  * tagMap                  1005    1235     109       121
  * tracerImmutableMap      1052    1249     123        -   (MapN)
  * stringIndex             1366    1724       -        -
- * stringIndex_embedded    1479    1846       -        -   (fastest get)
+ * stringIndex_embedded    1479    1846       -        -
  * }
      * - *

      Key findings: + *

      In this run: * *

        - *
      • StringIndex-as-map is a reliable {@code get} win, and it holds up under type-profile - * pollution: {@code stringIndex_embedded} (the {@code static final}-array form) and {@code - * stringIndex} (the instance wrapper) both beat every {@code Map} here by a wide margin, on - * both the {@code equals()} and identity-fast-path lookups. Unlike {@link - * ImmutableSetBenchmark}'s {@code hitFresh} case, there's no bimodality here — this win is - * unconditional, not access-pattern-dependent. - *
      • This table still compares boxed {@code Integer} values ({@code hashMap}/{@code - * linkedHashMap}/{@code treeMap}/{@code tracerImmutableMap} all return {@code Integer}; - * {@code tagMap}/{@code stringIndex} happen to expose primitive {@code int} accessors, but - * that's not exercised as a differentiator here). StringIndex's edge should widen further - * against a {@code Map} once the comparison is against genuinely autoboxed - * reads on both sides — not yet measured. - *
      • {@code treeMap}'s {@code get} has a wide error bar (±129, one fork's measurement iterations - * dropped to ~230-300, the rest sit around 490-610) — a known {@code TreeMap} comparison - * characteristic (uses {@code compareTo} not {@code hashCode}/{@code equals}, so it's - * unaffected by dispatch pollution), not the same warmup-race bimodality investigated in - * {@link ImmutableSetBenchmark}. - *
      • {@code TagMap.forEach} (121) beats its own {@code iterator} (109) by ~10%: TagMap's - * structure makes a faithful external {@code Iterator} expensive (externalized cursor + - * skip-empty + per-call re-entry + the iterator allocation) — all of which internal {@code - * forEach} avoids. Traverse TagMap via {@code forEach}, never its iterator; that gap only - * widens as TagMap's entry model grows. + *
      • The embedded StringIndex has the fastest {@code get}; the instance wrapper is second. + *
      • Both StringIndex variants outperform the map-based alternatives for distinct and identical + * key instances. + *
      • {@code TreeMap.get} varies widely across forks (roughly 230-610 M ops/s), so its mean is + * less stable than the other results. + *
      • {@code TagMap.forEach} is about 10% faster than its iterator (121 vs. 109 M ops/s). Its + * advantage widens as TagMap's entry model grows. *
      */ // @Fork(5): get_tracerImmutableMap* (MapN reached via interface dispatch) is JIT-bimodal at fewer diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java index ce5e7dc5ee2..b0977dc5ecb 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java @@ -44,75 +44,41 @@ * indirection cost of the wrapper. * * - *

      Hit lookups come in two flavors, because reusing the exact same key instances for both - * building a structure and measuring lookups against it is its own validity bug, independent of - * hash-dispatch pollution: {@link String#equals} takes an {@code ==} fast path, and {@link - * String#hashCode()} caches its result in a field on first call, so a key instance that was already - * inserted (or previously looked up) pays neither real cost again. + *

      Lookup variants: * *

        - *
      • {@code hit} -- looks up the same interned {@link #STRINGS} literals used to build every - * structure. This is realistic and worth keeping (fixed header/config-key lookups commonly - * are interned literals), but identity with the stored element is inherent to interning, not - * a choice this benchmark makes -- two equal literals are always the same instance. - *
      • {@code hitFresh} -- looks up {@link #FRESH_STRINGS}, separate {@code new String(..)} - * instances never touched by {@link #setUp}, so each carries an uncached hash and forces a - * real {@code equals()} beyond the {@code ==} check. Models keys arriving from - * parsing/concatenation/deserialization rather than literals. Only meaningful for the - * hash-based structures ({@code hashSet}, {@code tracerImmutableSet}, {@code stringIndex}, - * {@code stringIndex_embedded}); not measured for {@code array}/{@code sortedArray}/{@code - * treeSet}. + *
      • {@code hit} uses the same interned strings that were inserted, exercising the identity fast + * path. + *
      • {@code hitFresh} uses equal, non-interned strings, avoiding the identity fast path. It is + * measured only for the hash-based structures. + *
      • {@code miss} uses non-interned strings that are not in the set. *
      * - *

      Misses are already representative of both effects for free: {@link #MISSES} is built via - * concatenation (never interned) and never touched by {@link #setUp}. - * - *

      Full re-run, all six structures across {@code hit}/{@code hitFresh}/{@code miss} together, - * with {@link BenchmarkUtils#polluteHashDispatch()} in effect (Apple M1, Java 8u382 -- the - * repo-default {@code jmh} test launcher, no {@code -PtestJvm} override --, {@code @Fork(5)}, - * {@code @Threads(8)}; M ops/s = millions): + *

      Results on an Apple M1 with Java 8u382, {@link BenchmarkUtils#polluteHashDispatch()} enabled, + * {@code @Fork(5)}, and {@code @Threads(8)} (M ops/s): * * 

      {@code
  * Structure                    hit   hitFresh    miss
- * stringIndex_embedded (static) 2098      1563    2030    (fastest hit and miss)
+ * stringIndex_embedded (static) 2098      1563    2030
  * hashSet                       1723      1276    1823
- * stringIndex (inst)            1883      1184 *  1700 *  (* bimodal -- see caveat)
- * tracerImmutableSet            1632      1232    1625    (Set.copyOf / SetN)
+ * stringIndex (inst)            1883      1184 *  1700 *
+ * tracerImmutableSet            1632      1232    1625    (SetN)
  * array                          854         -     495
  * sortedArray                    713         -     613
  * treeSet                        646         -     544
  * }
      * - *

      Key findings: + *

      In this run: * *

        - *
      • The static {@code EmbeddingSupport} path is the fastest on both hit and miss -- it beats - * {@code HashSet} on both and crushes the scan/search/tree forms. - *
      • {@code stringIndex} (the instance wrapper) trails {@code EmbeddingSupport} by the - * field-load indirection. It reliably beats {@code HashSet} on {@code hit} (its actual design - * case: repeated lookups of a known, fixed name set). On {@code miss} and {@code hitFresh} it - * is not a reliable win -- both are bimodal across forks (see caveat below) and the - * mean in each case already sits at or below {@code HashSet}'s steady figure. For miss- or - * fresh-key-heavy membership use, prefer {@link StringIndex.EmbeddingSupport} directly rather - * than assuming the wrapper is strictly better than a plain {@code Set}. This doesn't apply - * to {@code StringIndex}'s parallel-value (map) use case ({@code mapValues}/{@code lookup}) - * -- that win comes from avoiding boxing and node overhead entirely and is unaffected by any - * of this. - *
      • {@link java.util.Set#copyOf} ({@code SetN}, the agent's compact fixed-set form) trails - * {@code EmbeddingSupport} on every scenario but remains the most compact (~27% - * smaller -- no cached hashes, no 2x table). So StringIndex's edge over {@code SetN} is speed - * + the {@code indexOf}->parallel-array capability, not footprint. - *
      • {@code array} / {@code sortedArray} / {@code treeSet} trail every hashed structure, most on - * miss. - *
      • {@code hitFresh} is the slowest of the three scenarios for every hash-based - * structure -- clearly below both {@code hit} and {@code miss}, not merely below {@code hit} - * as previously guessed. This makes sense once the two failure shapes are compared: a miss - * usually short-circuits on the first hash mismatch during probing and rarely reaches {@code - * equals()}, while a {@code hitFresh} lookup must probe until it finds the match and pay a - * real, uncached {@code equals()} there -- so it is not simply "the honest version of hit", - * it exercises a genuinely more expensive path than either {@code hit} (cached hash + {@code - * ==}) or {@code miss} (hash-only rejection). Superseded an earlier partial-data guess that - * {@code hitFresh} would land at the same cost as {@code miss}. + *
      • The embedded {@code StringIndex} is fastest for all three lookup variants. + *
      • The {@code StringIndex} wrapper beats {@code HashSet} for interned hits. Its fresh-hit and + * miss results are bimodal and have lower means than {@code HashSet}; prefer the embedded + * form when these paths matter. + *
      • {@code SetN} is slower than the embedded form but about 27% smaller. StringIndex trades + * that space for speed and support for slot-aligned payload arrays. + *
      • Fresh hits are slower than misses for each hash-based structure: a matching distinct string + * reaches {@code equals()}, while a miss can stop on a hash mismatch. *
      * *

      Caveat — the instance {@code stringIndex} miss is bimodal across forks (confirmed at @@ -138,11 +104,7 @@ public class ImmutableSetBenchmark { /** Distinct String instances that are never present, for the miss path. */ static final String[] MISSES = newMisses(); - /** - * Equal-content, non-interned, never-before-hashed copies of {@link #STRINGS}, built once here - * and never touched by {@link #setUp} -- so a lookup against them can't ride the {@code ==} fast - * path or a hash cached during set construction. See {@code hitFresh} in the class javadoc. - */ + /** Equal, non-interned copies of {@link #STRINGS} used to exercise equality. */ static final String[] FRESH_STRINGS = newFreshStrings(); static String[] newFreshStrings() { diff --git a/internal-api/src/main/java/datadog/trace/util/StringIndex.java b/internal-api/src/main/java/datadog/trace/util/StringIndex.java index 66644e0fecf..d5f848b6f41 100644 --- a/internal-api/src/main/java/datadog/trace/util/StringIndex.java +++ b/internal-api/src/main/java/datadog/trace/util/StringIndex.java @@ -334,7 +334,11 @@ public static int indexOf(int[] hashes, String[] names, String name) { return indexOf(hashes, names, name, hash(name)); } - /** {@code indexOf(hashes, names, name) >= 0}. Mirrors {@link StringIndex#contains}. */ + /** + * Mirrors {@link StringIndex#contains}. + * + * @return {@code true} when {@code name} is present in the index + */ public static boolean contains(int[] hashes, String[] names, String name) { return indexOf(hashes, names, name) >= 0; } From b5e900789efa5264175cca81eabd7de1d38e8713 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 20:28:48 -0400 Subject: [PATCH 08/52] Extend BenchmarkUtils with ConcurrentHashMap and Map-dispatch pollution polluteHashDispatch() only warmed the shared HashSet/HashMap call site; ConcurrentHashMap has its own, unrelated hashCode()/equals() dispatch sites and needs a dedicated scratch instance. Also add populateTypeProfileMap/populateTypeProfileMutableMap as the Map counterparts to the existing Collection-based helpers, for benchmarks that need finer-grained control than polluteHashDispatch() gives. --- .../datadog/trace/util/BenchmarkUtils.java | 71 +++++++++++++++++-- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java index 4aa5ea7556a..7e2fd2d2ec2 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java +++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java @@ -3,6 +3,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashSet; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** Shared setup helpers for JMH benchmarks in this module. */ public final class BenchmarkUtils { @@ -13,14 +15,41 @@ private BenchmarkUtils() {} }; /** - * Makes the internal {@code hashCode()} and {@code equals()} call sites of common hash-based - * collections megamorphic before measurement. + * Exercises {@link HashSet}/{@link java.util.HashMap}, the tracer's {@link + * CollectionUtils#tryMakeImmutableSet} immutable sets, and {@link ConcurrentHashMap} with several + * distinct key classes, so each structure's internal {@code hashCode()}/{@code equals()} dispatch + * -- a call site shared JVM-wide by every instance of that structure in the process, regardless + * of which specific instance or call site invokes {@code add}/{@code contains}/{@code get} -- is + * already megamorphic before a benchmark measures lookups against a single key type. * - *

      HotSpot records receiver classes at each virtual call site. A benchmark using only {@code - * String} keys leaves the sites inside these shared implementations monomorphic, allowing C2 to - * devirtualize and inline them. Production uses many key types, so the same sites are often - * megamorphic and retain virtual dispatch. Exercising several key classes avoids reporting - * unrealistically fast collection lookups. + *

      This matches production: those shared internal call sites are hit by every hash-based + * structure in the JVM across whatever key types the whole application uses, so they're + * realistically almost always megamorphic. An isolated benchmark that only ever looks up one key + * type (e.g. {@code String}) would otherwise leave them artificially monomorphic for the entire + * run, understating real dispatch cost. + * + *

      {@code HashSet} is backed by {@code HashMap} in the JDK, so polluting it also covers plain + * {@code HashMap} and {@code LinkedHashMap} (which extends {@code HashMap}) -- they share the + * same internal dispatch call site. {@code ConcurrentHashMap} does not: it's an unrelated class + * with its own {@code hashCode()}/{@code equals()} call sites, so it needs its own scratch + * instance (also covers {@code ConcurrentHashMap#newKeySet()}, which is backed by a {@code + * ConcurrentHashMap}). Structures that dispatch on {@code compareTo} instead ({@code TreeMap}, + * {@code TreeSet}, {@code ConcurrentSkipListMap}) aren't affected by any of this and don't need + * pollution. + * + *

      Deliberately does not touch the benchmark's own {@code contains}/{@code add}/{@code get} + * call sites -- those are realistically free to specialize per caller, the way a genuinely hot, + * narrowly-typed call site would in production. + * + *

      Not to be confused with the CHA-defeat decoys in {@code SingleThreadedMapBenchmark}/{@code + * ThreadSafeMapBenchmark} ({@code KeyStrategy} implementors referenced only so they're loaded, + * never invoked): that technique denies class-hierarchy analysis a single-implementor bet for a + * narrow, dd-trace-java-owned interface, and works by class-loading alone. It doesn't apply here + * -- {@code Object.hashCode()}/{@code equals()} already have countless implementors loaded in any + * real JVM, so a single-implementor CHA bet was never available for them. What gates their + * dispatch is the interpreter's per-call-site type profile, which only invocation can pollute -- + * hence this helper actually calls {@code add}/{@code contains}/{@code get}, rather than just + * loading classes. */ public static void polluteHashDispatch() { polluteHashDispatch(DEFAULT_DECOY_KEYS); @@ -29,6 +58,7 @@ public static void polluteHashDispatch() { public static void polluteHashDispatch(Object... decoyKeys) { populateTypeProfileMutable(new HashSet<>(), decoyKeys); populateTypeProfile(CollectionUtils.tryMakeImmutableSet(Arrays.asList(decoyKeys)), decoyKeys); + populateTypeProfileMutableMap(new ConcurrentHashMap<>(), decoyKeys); } /** @@ -56,4 +86,31 @@ public static void populateTypeProfileMutable(Collection scratch, Object scratch.contains(key); } } + + /** + * {@link Map} counterpart to {@link #populateTypeProfile(Collection)}: pass the map instance + * under test (or an equivalent scratch instance) to drive its {@code get()} dispatch. Safe + * against immutable maps too, since it only calls {@code get()}. + */ + public static void populateTypeProfileMap(Map populated) { + populateTypeProfileMap(populated, DEFAULT_DECOY_KEYS); + } + + public static void populateTypeProfileMap(Map populated, Object... decoyKeys) { + for (Object key : decoyKeys) { + populated.get(key); + } + } + + /** + * Lower-level control, {@link Map} counterpart to {@link #populateTypeProfileMutable}: also + * drives {@code put()} dispatch, so {@code scratch} must genuinely support mutation. + */ + public static void populateTypeProfileMutableMap( + Map scratch, Object... decoyKeys) { + for (Object key : decoyKeys) { + scratch.put(key, key); + scratch.get(key); + } + } } From f2b6e86c1085e2b7e1175d38617554afd09a53f0 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 20:29:00 -0400 Subject: [PATCH 09/52] Wire BenchmarkUtils.polluteHashDispatch into the remaining map/set benchmarks Applies the type-profile-pollution fix already used in ImmutableSetBenchmark/ImmutableMapBenchmark to the rest of the HashMap/HashSet/ConcurrentHashMap-comparing benchmarks in this module: SingleThreadedMapBenchmark, SingleThreadedSetBenchmark, ThreadSafeMapBenchmark, HashtableD1Benchmark, HashtableD2Benchmark, CaseInsensitiveMapBenchmark, and TagMapAccessBenchmark. Without this, each file's isolated single-key-type usage left the JDK collections' shared internal hashCode()/equals() dispatch call sites artificially monomorphic, understating their real per-call cost. FlatHashtableIteratorBenchmark is intentionally untouched -- it only exercises the project's own FlatHashtable/HashStrategy, not java.util.HashMap/HashSet/ConcurrentHashMap. --- .../jmh/java/datadog/trace/api/TagMapAccessBenchmark.java | 6 ++++++ .../datadog/trace/util/CaseInsensitiveMapBenchmark.java | 7 +++++++ .../jmh/java/datadog/trace/util/HashtableD1Benchmark.java | 2 ++ .../jmh/java/datadog/trace/util/HashtableD2Benchmark.java | 2 ++ .../datadog/trace/util/SingleThreadedMapBenchmark.java | 2 ++ .../datadog/trace/util/SingleThreadedSetBenchmark.java | 2 ++ .../datadog/trace/util/ThreadSafeMapCounterBenchmark.java | 1 + .../java/datadog/trace/util/ThreadSafeMapD1Benchmark.java | 1 + .../java/datadog/trace/util/ThreadSafeMapD2Benchmark.java | 1 + 9 files changed, 24 insertions(+) diff --git a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java index de33c957e9a..28d14e44718 100644 --- a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java @@ -1,5 +1,6 @@ package datadog.trace.api; +import datadog.trace.util.BenchmarkUtils; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -99,6 +100,11 @@ public class TagMapAccessBenchmark { * Pre-populated read map, PER-THREAD ({@code Scope.Thread}): each thread owns its own map so * reads don't contend on shared mutable state under {@code @Threads(8)}. */ + @Setup(Level.Trial) + public void setUp() { + BenchmarkUtils.polluteHashDispatch(); + } + @State(Scope.Thread) public static class ReadMap { TagMap map; diff --git a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java index 9cbbddcf299..e99080539b7 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java @@ -6,8 +6,10 @@ import java.util.function.Supplier; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; @@ -101,6 +103,11 @@ static T init(Supplier supplier) { // masking exactly the differences this benchmark compares. int lookupIndex = 0; + @Setup(Level.Trial) + public void setUp() { + BenchmarkUtils.polluteHashDispatch(); + } + String nextLookupKey() { int localIndex = ++lookupIndex; if (localIndex >= LOOKUP_KEYS.length) { diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java index f8ba7177e88..b8c8988d93a 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java @@ -103,6 +103,8 @@ public static class D1State { @Setup(Level.Iteration) public void setUp() { + BenchmarkUtils.polluteHashDispatch(); + table = new Hashtable.D1<>(CAPACITY); hashMap = new HashMap<>(CAPACITY); keys = SOURCE_KEYS; diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java index 6f46a702005..55258135180 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java @@ -138,6 +138,8 @@ public static class D2State { @Setup(Level.Iteration) public void setUp() { + BenchmarkUtils.polluteHashDispatch(); + table = new Hashtable.D2<>(CAPACITY); hashMap = new HashMap<>(CAPACITY); k1s = SOURCE_K1; diff --git a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java index cb792cc1ca9..02295a46edb 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java @@ -191,6 +191,8 @@ static IntEntry[] newFilledFlat() { @Setup(Level.Trial) public void setUp() { + BenchmarkUtils.polluteHashDispatch(); + hashMap = new HashMap<>(); fill(hashMap); synchronizedHashMap = Collections.synchronizedMap(new HashMap<>(hashMap)); diff --git a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java index e145e6bbe8b..1825771f852 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java @@ -94,6 +94,8 @@ static void fill(Set set) { @Setup(Level.Trial) public void setUp() { + BenchmarkUtils.polluteHashDispatch(); + hashSet = new HashSet<>(Arrays.asList(ELEMENTS)); synchronizedSet = Collections.synchronizedSet(new HashSet<>(hashSet)); treeSet = new TreeSet<>(Arrays.asList(ELEMENTS)); diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java index bc8bc07b9f5..ac5cdbe3bd5 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java @@ -100,6 +100,7 @@ public static class SharedState { @Setup(Level.Iteration) public void setUp() { + BenchmarkUtils.polluteHashDispatch(); table = ConcurrentHashtable.D1.createBounded(CounterEntry.class, CAPACITY); atomicLongMap = new ConcurrentHashMap<>(CAPACITY); longAdderMap = new ConcurrentHashMap<>(CAPACITY); diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java index 43d9b849f48..bf51916eef6 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java @@ -104,6 +104,7 @@ public static class SharedState { @Setup(Level.Iteration) public void setUp() { + BenchmarkUtils.polluteHashDispatch(); table = ConcurrentHashtable.D1.createBounded(D1Entry.class, CAPACITY); concurrentHashMap = new ConcurrentHashMap<>(CAPACITY); skipListMap = new ConcurrentSkipListMap<>(); diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java index 57506a12230..06e079086cb 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java @@ -184,6 +184,7 @@ public static class SharedState { @Setup(Level.Iteration) public void setUp() { + BenchmarkUtils.polluteHashDispatch(); table = ConcurrentHashtable.D2.createBounded(D2Entry.class, CAPACITY); supportBuckets = ConcurrentHashtable.createFixedBuckets(SupportEntry.class, CAPACITY); concurrentHashMap = new ConcurrentHashMap<>(CAPACITY); From dd3eba5fa1ccf9556e3e3642ad48970ef373a1f0 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 22:38:30 -0400 Subject: [PATCH 10/52] Record pollution-corrected results in SingleThreadedMapBenchmark --- .../util/SingleThreadedMapBenchmark.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java index 02295a46edb..26753867d18 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java @@ -50,6 +50,48 @@ * unsynchronized {@code hashMap} {@code get}/{@code iterate} methods are the in-harness baseline; * the tax is the delta to the {@code synchronizedHashMap} equivalents. Comparing across JVM * versions at stock flags shows the biased-locking effect. (Results pending a fresh multi-JVM run.) + * + *

      JDK 8 results (Apple M1, {@code @Fork(2)}, {@code @Threads(8)}, with {@link + * BenchmarkUtils#polluteHashDispatch()} in effect; M ops/s): + * + * 

      {@code
+ * create_hashMap                 79   create_hashMap_sized     38*
+ * create_synchronizedHashMap    8.6   create_treeMap            5*
+ * create_linkedHashMap            8*  create_tagMap            10
+ * create_tagMap_via_ledger        9   create_flatHashtable    190
+ *
+ * clone_hashMap                  68*  clone_synchronizedHashMap 58
+ * clone_treeMap                 100   clone_linkedHashMap       94
+ * clone_tagMap                  249
+ *
+ * get_hashMap                   164   get_synchronizedHashMap   67
+ * get_flatHashtable              196
+ *
+ * iterate_hashMap                119  iterate_synchronizedHashMap 81
+ * iterate_flatHashtable           14*
+ * }
      + * + *

      * = error bar as wide as (or wider than) the mean at {@code @Fork(2)} — treat these as + * directional, not decisive; a {@code @Fork(5)} rerun would tighten them (see {@code + * ThreadSafeMapBenchmark}'s Javadoc for the same caveat pattern). The construction benchmarks are + * consistently the noisy ones; the read/clone/iterate benchmarks are comparatively tight. + * + *

      Key findings: + * + *

        + *
      • {@code flatHashtable} dominates both {@code create} (190M) and {@code get} (196M) — the + * unboxed, self-contained entry and comparison-free insert pay off, consistent with every + * other FlatHashtable comparison in this module. + *
      • {@code tagMap} clone (249M) is ~3.7x {@code hashMap} clone (68M) — the same story {@link + * datadog.trace.api.TagMapAccessBenchmark} reports from an earlier (unpolluted, Java 17) run + * at ~4.6x; the ratio survives pollution and a different JDK, even though the absolute + * numbers aren't directly comparable across those two runs. + *
      • The uncontended synchronization tax is large here even though this run is on JDK 8, where + * biased locking is enabled by default: {@code get_hashMap} (164M) → {@code + * get_synchronizedHashMap} (67M) is a ~59% hit, and {@code iterate} (119M → 81M) is ~32%. + * That's a bigger tax than the "biased locking should make uncontended locking nearly free" + * story predicts — not root-caused here, left as an open question rather than papered over. + *
      */ @Fork(2) @Warmup(iterations = 2) From 8c60545676af26444382269c741f58c43fc91ffe Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 22:38:32 -0400 Subject: [PATCH 11/52] Record pollution-corrected results in SingleThreadedSetBenchmark --- .../util/SingleThreadedSetBenchmark.java | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java index 1825771f852..2b19e3ac48f 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java @@ -45,16 +45,36 @@ * create_treeSet 36 * } * + *

      JDK 8 results (Apple M1, {@code @Fork(2)}, {@code @Threads(8)}, with {@link + * BenchmarkUtils#polluteHashDispatch()} in effect; M ops/s): + * + * 

      {@code
+ * contains_hashSet            1421
+ * contains_synchronizedSet     746    (~48% slower — the uncontended sync tax)
+ * iterate_hashSet              134
+ * iterate_synchronizedSet      129    (one monitor acquire amortized over the walk)
+ *
+ * create_hashSet         67    clone_hashSet          56
+ * create_hashSet_sized   83*   clone_synchronizedSet  48*
+ * create_linkedHashSet   63    clone_linkedHashSet    56
+ * create_synchronizedSet 71*   clone_treeSet          77
+ * create_treeSet         38
+ * }
      + * + *

      * = error bar over half the mean at {@code @Fork(2)} — directional only. + * *

      Key findings: * *

        - *
      • Uncontended synchronization tax on {@code contains} is ~37% (1291 → 808M ops/s) even - * with no contention and biased locking disabled (Java 17, JEP 374) — the full per-lock CAS - * cost. On {@code iterate} it nearly vanishes: a single monitor acquire amortized over the - * traversal. - *
      • Construction: {@code TreeSet} is the slowest to build (~36M); the {@code synchronizedSet} - * wrapper adds a modest cost over plain {@code HashSet}. (Allocation-path numbers carry more - * run-to-run variance than the read paths.) + *
      • Uncontended synchronization tax holds up under pollution and on a different JDK: + * {@code contains} is ~48% slower synchronized (1421 → 746M ops/s on JDK 8, vs. ~37% on Java + * 17) — same story, somewhat larger tax. {@code iterate}'s tax stays small either way (~4% + * here): one monitor acquire amortized over the walk. + *
      • Type-profile pollution didn't change the qualitative story from the original Java 17 run — + * {@code contains_hashSet} and {@code iterate_hashSet} land in the same range (1291 vs 1421M, + * 91 vs 134M) rather than collapsing, unlike {@link ImmutableSetBenchmark}'s {@code hitFresh} + * case. Construction numbers remain the noisiest (several {@code @Fork(2)} error bars exceed + * half the mean); {@code TreeSet} stays the slowest to build across both runs. *
      */ @Fork(2) From 0aea1167c7033361a11ae290fd352e0309bea140 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 22:38:34 -0400 Subject: [PATCH 12/52] Record pollution rerun results in HashtableD1Benchmark --- .../java/datadog/trace/util/HashtableD1Benchmark.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java index b8c8988d93a..8653ac46fb7 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java @@ -54,6 +54,16 @@ * HashtableD1Benchmark.iterate_hashMap thrpt 6 20.043 ± 0.752 ops/us * HashtableD1Benchmark.iterate_hashtable thrpt 6 22.208 ± 0.956 ops/us * + * + *

      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} added to {@code D1State.setUp()} (same + * machine/JVM/config): every number moved down somewhat (add_hashMap 188→101, update_hashtable + * 1810→1465, iterate_hashtable 22→17 ops/us), including {@code *_hashtable}, which doesn't touch + * {@code java.util.HashMap}/{@code HashSet} at all and so shouldn't be affected by this pollution + * mechanism. That points to session-to-session machine variance (not controlled for here) rather + * than a genuine pollution effect for this particular file — unlike {@code + * ImmutableSetBenchmark}/{@code ImmutableMapBenchmark}, where pollution measurably changed the + * comparison. The relative conclusion (D1 dominates {@code update}, is roughly comparable on + * {@code add}, ties on {@code iterate}) is unchanged either way. */ @Fork(2) @Warmup(iterations = 2) From fe50efdd092e8029c332e2475e87fe2da8637051 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 22:38:35 -0400 Subject: [PATCH 13/52] Record pollution rerun results in HashtableD2Benchmark --- .../java/datadog/trace/util/HashtableD2Benchmark.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java index 55258135180..c2abfa6726c 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java @@ -58,6 +58,17 @@ * HashtableD2Benchmark.iterate_hashMap thrpt 6 19.508 ± 0.760 ops/us * HashtableD2Benchmark.iterate_hashtable thrpt 6 16.968 ± 0.371 ops/us * + * + *

      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} added to {@code D2State.setUp()} (same + * machine/JVM/config): results were noisy and inconsistent with a clean pollution story — + * add_hashMap actually rose (77→103), while add_hashtable fell sharply (217→118, error bars wider + * than the mean both times); update_hashtable fell (1446→1225) and both iterate numbers fell + * (19.5→15.4, 17.0→13.1) despite {@code *_hashtable} not touching {@code java.util.HashMap}/{@code + * HashSet} dispatch at all. As with {@link HashtableD1Benchmark}, this looks like uncontrolled + * machine variance between the two sessions rather than a genuine pollution effect here — treat + * these two runs as not directly comparable. The relative conclusion (D2 dominates {@code + * update}, wins {@code add} by avoiding the {@code Key2} allocation, ties on {@code iterate}) is + * unchanged either way. */ @Fork(2) @Warmup(iterations = 2) From b6835850669b78b9466ca6fc2917b2bc40b511db Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 22:38:36 -0400 Subject: [PATCH 14/52] Record pollution rerun results in CaseInsensitiveMapBenchmark --- .../util/CaseInsensitiveMapBenchmark.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java index e99080539b7..0368f4fd516 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java @@ -54,6 +54,30 @@ * lookup_hashMap thrpt 15 441875038.1 ± 110408182.2 ops/s 24.0 B/op (129 GCs) * lookup_treeMap thrpt 15 251195415.1 ± 14662568.3 ops/s ~0 B/op * + * + *

      Rerun on JDK 8 with {@link BenchmarkUtils#polluteHashDispatch()} added to a new + * {@code @Setup(Level.Trial)} (this file had none before), at this file's actual {@code @Fork(2)} + * (the numbers above are from an ad hoc higher-fork run; not directly comparable). M ops/s, 8 + * threads: + * + * 

      {@code
+ * create_baseline        26    create_flatHashtable   13
+ * create_hashMap          9    create_treeMap          7
+ *
+ * lookup_baseline      2618    lookup_flatHashtable  415
+ * lookup_flatHashtable_lowLoad 415  lookup_hashMap    367*
+ * lookup_treeMap        209
+ * }
      + * + *

      * = error bar over a third of the mean at {@code @Fork(2)} — directional only. + * + *

      All four {@code lookup_*} numbers sit 17-23% below the table above (415 vs 537 flatHashtable, + * 367 vs 442 hashMap, 209 vs 251 treeMap) despite {@code flatHashtable} and {@code treeMap} using + * neither {@code java.util.HashMap} nor {@code hashCode()}/{@code equals()} dispatch — so this drop + * isn't attributable to pollution. Combined with the same pattern in {@link HashtableD1Benchmark} + * and {@link HashtableD2Benchmark}, this looks like session-to-session machine variance (different + * JDK, different run) rather than a real regression. The relative ranking — {@code + * flatHashtable} > {@code hashMap} > {@code treeMap} — is unchanged. */ @Fork(2) @Warmup(iterations = 2) From c514c05436faa28da6f9eb2ce7d866e6d3353926 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 25 Aug 2026 22:38:37 -0400 Subject: [PATCH 15/52] Record pollution rerun results in TagMapAccessBenchmark --- .../trace/api/TagMapAccessBenchmark.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java index 28d14e44718..d373a6a5e18 100644 --- a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java @@ -57,6 +57,28 @@ * TagMapAccessBenchmark.insert_hashMap_builderStyle thrpt 5 28057827.189 ± 1359655.664 ops/s * TagMapAccessBenchmark.insert_via_ledger thrpt 5 41169656.095 ± 773264.754 ops/s * + * + *

      Rerun on JDK 8 with a new top-level {@code @Setup(Level.Trial)} calling {@link + * BenchmarkUtils#polluteHashDispatch()} (this file had none before). M ops/s, 8 threads: + * + * 

      {@code
+ * getEntry                        83   getObject                    87
+ * insert                          37   insert_hashMap                48
+ * insert_hashMap_builderStyle     20   insert_via_ledger             37*
+ * }
      + * + *

      * = error bar about a quarter of the mean at {@code @Fork(2)} — directional only. + * + *

      Every number here is 9-29% below the Java 17 table above, with no clear split between the + * TagMap paths and the HashMap paths this pollution should affect — the same broad slowdown pattern + * seen across {@link datadog.trace.util.HashtableD1Benchmark}, {@link + * datadog.trace.util.HashtableD2Benchmark}, and {@link + * datadog.trace.util.CaseInsensitiveMapBenchmark} in the same session, so treat it as + * session-to-session machine/JDK variance rather than a pollution-driven regression. The relative + * story survives: {@code insert_hashMap} (48M) still beats {@code insert} (37M) for plain + * insertion, and {@code insert_via_ledger} (37M) still clearly beats the HashMap builder-style path + * (20M); {@code insert_via_ledger} landing roughly level with {@code insert} here (vs. clearly + * behind it in the table above) is within that path's own wide error bar, not a new finding. */ @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.SECONDS) From 6749897ed9656dd7fcfff64305c24009f4a393ba Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 26 Aug 2026 07:56:21 -0400 Subject: [PATCH 16/52] Correct HashtableD1Benchmark's pollution-immunity rationale D1.Entry.hash/matches are call sites private to Hashtable.java, distinct from java.util.HashMap/HashSet's own dispatch sites, so pollution literally cannot reach them (not just unlikely to). Since JDK and machine were held constant in this rerun, attribute the observed drop to same-session run-to-run noise, not a JDK effect (that explanation applies elsewhere, where the JDK actually changed). --- .../trace/util/HashtableD1Benchmark.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java index 8653ac46fb7..587eaad3da2 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java @@ -57,13 +57,18 @@ * *

      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} added to {@code D1State.setUp()} (same * machine/JVM/config): every number moved down somewhat (add_hashMap 188→101, update_hashtable - * 1810→1465, iterate_hashtable 22→17 ops/us), including {@code *_hashtable}, which doesn't touch - * {@code java.util.HashMap}/{@code HashSet} at all and so shouldn't be affected by this pollution - * mechanism. That points to session-to-session machine variance (not controlled for here) rather - * than a genuine pollution effect for this particular file — unlike {@code - * ImmutableSetBenchmark}/{@code ImmutableMapBenchmark}, where pollution measurably changed the - * comparison. The relative conclusion (D1 dominates {@code update}, is roughly comparable on - * {@code add}, ties on {@code iterate}) is unchanged either way. + * 1810→1465, iterate_hashtable 22→17 ops/us), including {@code *_hashtable}. That's expected to be + * a no-op for {@code *_hashtable}: {@link Hashtable.D1.Entry#hash} and {@link + * Hashtable.D1.Entry#matches} are call sites private to {@code Hashtable.java}, structurally + * distinct from {@code java.util.HashMap}/{@code HashSet}'s internal {@code hashCode()}/{@code + * equals()} call sites — JIT type profiles are keyed per call site, so {@code + * polluteHashDispatch()} cannot reach them regardless of key-type overlap. Since the JDK and + * machine were held constant across this rerun (unlike the JDK 8-vs-17 comparisons in {@link + * datadog.trace.util.CaseInsensitiveMapBenchmark} and {@link + * datadog.trace.api.TagMapAccessBenchmark}), the drop here is same-session run-to-run noise + * (thermal/power, not controlled for) rather than either a pollution effect or a JDK effect. The + * relative conclusion (D1 dominates {@code update}, is roughly comparable on {@code add}, + * ties on {@code iterate}) is unchanged either way. */ @Fork(2) @Warmup(iterations = 2) From 35c35910b2bdca99187ca84ec8ec2311c9ad4518 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 26 Aug 2026 07:56:30 -0400 Subject: [PATCH 17/52] Correct HashtableD2Benchmark's pollution-immunity rationale Same fix as HashtableD1Benchmark: D2.Entry.hash/matches are call sites private to Hashtable.java, structurally distinct from java.util.HashMap dispatch, so pollution cannot reach them regardless of key-type overlap; the observed drop with JDK/machine held constant is same-session noise, not a JDK effect. --- .../datadog/trace/util/HashtableD2Benchmark.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java index c2abfa6726c..7c9e412d479 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java @@ -63,12 +63,15 @@ * machine/JVM/config): results were noisy and inconsistent with a clean pollution story — * add_hashMap actually rose (77→103), while add_hashtable fell sharply (217→118, error bars wider * than the mean both times); update_hashtable fell (1446→1225) and both iterate numbers fell - * (19.5→15.4, 17.0→13.1) despite {@code *_hashtable} not touching {@code java.util.HashMap}/{@code - * HashSet} dispatch at all. As with {@link HashtableD1Benchmark}, this looks like uncontrolled - * machine variance between the two sessions rather than a genuine pollution effect here — treat - * these two runs as not directly comparable. The relative conclusion (D2 dominates {@code - * update}, wins {@code add} by avoiding the {@code Key2} allocation, ties on {@code iterate}) is - * unchanged either way. + * (19.5→15.4, 17.0→13.1). As with {@link HashtableD1Benchmark}, {@code *_hashtable} is expected to + * be a no-op here: {@link Hashtable.D2.Entry#hash} and {@link Hashtable.D2.Entry#matches} are call + * sites private to {@code Hashtable.java}, structurally distinct from {@code + * java.util.HashMap}/{@code HashSet}'s internal dispatch call sites — pollution cannot reach them. + * With the JDK and machine held constant across this rerun, the drop is same-session run-to-run + * noise (thermal/power, not controlled for) rather than a genuine pollution effect. Treat these two + * runs as not directly comparable on absolute numbers. The relative conclusion (D2 dominates + * {@code update}, wins {@code add} by avoiding the {@code Key2} allocation, ties on {@code + * iterate}) is unchanged either way. */ @Fork(2) @Warmup(iterations = 2) From f6e3ca825b19439d2d6f394775aa690181fa6426 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 26 Aug 2026 07:56:37 -0400 Subject: [PATCH 18/52] Attribute CaseInsensitiveMapBenchmark's rerun slowdown to JDK 8 on ARM64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original table is Zulu 21; this rerun is JDK 8, whose C2 backend for Apple Silicon is far less mature than JDK 17+'s. That JDK gap explains a broad-based slowdown across every entry on its own, including flatHashtable/treeMap which don't touch java.util.HashMap dispatch — a more precise explanation than generic "machine variance." --- .../datadog/trace/util/CaseInsensitiveMapBenchmark.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java index 0368f4fd516..62a48976691 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java @@ -74,10 +74,11 @@ *

      All four {@code lookup_*} numbers sit 17-23% below the table above (415 vs 537 flatHashtable, * 367 vs 442 hashMap, 209 vs 251 treeMap) despite {@code flatHashtable} and {@code treeMap} using * neither {@code java.util.HashMap} nor {@code hashCode()}/{@code equals()} dispatch — so this drop - * isn't attributable to pollution. Combined with the same pattern in {@link HashtableD1Benchmark} - * and {@link HashtableD2Benchmark}, this looks like session-to-session machine variance (different - * JDK, different run) rather than a real regression. The relative ranking — {@code - * flatHashtable} > {@code hashMap} > {@code treeMap} — is unchanged. + * isn't attributable to pollution. The likelier explanation: the table above is Zulu 21, this rerun + * is JDK 8, and JDK 8's C2 backend for Apple Silicon (AArch64) is far less mature than JDK 17+'s — + * a broad-based slowdown across every entry, pollution-affected or not, is expected from that JDK + * gap alone on this machine. The relative ranking — {@code flatHashtable} > {@code hashMap} + * > {@code treeMap} — is unchanged. */ @Fork(2) @Warmup(iterations = 2) From e6175c57f295deaff92b6da01b02e33ecb0da9b2 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 26 Aug 2026 07:56:45 -0400 Subject: [PATCH 19/52] Attribute TagMapAccessBenchmark's rerun slowdown to JDK 8 on ARM64 Same fix as CaseInsensitiveMapBenchmark: the original table is Java 17, this rerun is JDK 8, and JDK 8's weaker Apple Silicon C2 codegen explains the across-the-board drop on its own. Distinguish this from HashtableD1Benchmark/HashtableD2Benchmark, where the JDK was held constant and the drop is same-session noise instead. --- .../trace/api/TagMapAccessBenchmark.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java index d373a6a5e18..3cd021a65b4 100644 --- a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java @@ -70,15 +70,18 @@ *

      * = error bar about a quarter of the mean at {@code @Fork(2)} — directional only. * *

      Every number here is 9-29% below the Java 17 table above, with no clear split between the - * TagMap paths and the HashMap paths this pollution should affect — the same broad slowdown pattern - * seen across {@link datadog.trace.util.HashtableD1Benchmark}, {@link - * datadog.trace.util.HashtableD2Benchmark}, and {@link - * datadog.trace.util.CaseInsensitiveMapBenchmark} in the same session, so treat it as - * session-to-session machine/JDK variance rather than a pollution-driven regression. The relative - * story survives: {@code insert_hashMap} (48M) still beats {@code insert} (37M) for plain - * insertion, and {@code insert_via_ledger} (37M) still clearly beats the HashMap builder-style path - * (20M); {@code insert_via_ledger} landing roughly level with {@code insert} here (vs. clearly - * behind it in the table above) is within that path's own wide error bar, not a new finding. + * TagMap paths and the HashMap paths this pollution should affect. The table above is Java 17; this + * rerun is JDK 8, whose C2 backend for Apple Silicon (AArch64) is far less mature than JDK 17+'s — + * a broad-based slowdown across every entry is expected from that JDK gap alone, independent of + * pollution — the same JDK-crossing explanation applies to {@link + * datadog.trace.util.CaseInsensitiveMapBenchmark}'s rerun. ({@link + * datadog.trace.util.HashtableD1Benchmark} and {@link datadog.trace.util.HashtableD2Benchmark} saw + * a similar broad drop despite holding the JDK constant — that one is same-session run-to-run + * noise, not a JDK effect.) The relative story survives: {@code insert_hashMap} (48M) still beats + * {@code insert} (37M) for plain insertion, and {@code insert_via_ledger} (37M) still clearly beats + * the HashMap builder-style path (20M); {@code insert_via_ledger} landing roughly level with {@code + * insert} here (vs. clearly behind it in the table above) is within that path's own wide error bar, + * not a new finding. */ @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.SECONDS) From 7f9fbf149aadf7742f1b91afedccd90a36b2680d Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 26 Aug 2026 09:26:23 -0400 Subject: [PATCH 20/52] Record Java 17 rerun for HashtableD1Benchmark Controlled rerun (same machine, JDK swapped to Zulu 17.0.7) shows update_hashtable still wins decisively (~4.2x, down from ~14x on JDK 8) and iterate_hashtable flips from a wash to a ~4x win, while add_hashMap edges ahead slightly. Java 17's much better allocator/GC narrows but doesn't erase Hashtable's win on the allocation-heavy paths -- net takeaway: Hashtable is a strong HashMap substitute for simple counter/tally cases with a primitive value. Absolute numbers aren't comparable to the JDK 8 table: JMH auto-selects the cheap "compiler" Blackhole mode on 17 but not on 8, so within-run ratios are the only trustworthy comparison here. --- .../trace/util/HashtableD1Benchmark.java | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java index 587eaad3da2..9581a8db520 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java @@ -38,10 +38,14 @@ *

    • iterate — walk every entry and consume its key + value. * * - *

      Update is where Hashtable dominates: D1 is ~14x faster, because the HashMap path - * allocates per call (a {@code Long}) and the resulting GC pressure throttles throughput under - * multiple threads. Add is roughly comparable (both allocate one entry per insert). - * Iterate is essentially a wash — both are bucket walks. + *

      Update is where Hashtable dominates: D1 is ~14x faster on JDK 8 (see the Java 17 rerun + * below for a narrower but still decisive margin), because the HashMap path allocates per call (a + * {@code Long}) and the resulting GC pressure throttles throughput under multiple threads. This is + * the headline case for {@code Hashtable}: a simple counter/tally with a primitive value is exactly + * where HashMap's autoboxing tax bites hardest, and {@code Hashtable.D1} sidesteps it entirely by + * mutating a field on the retrieved entry in place. Add is roughly comparable (both allocate + * one entry per insert). Iterate is essentially a wash on JDK 8, though not on Java 17 (see + * below). * MacBook M1 8 threads (Java 8) * * Benchmark Mode Cnt Score Error Units @@ -69,6 +73,29 @@ * (thermal/power, not controlled for) rather than either a pollution effect or a JDK effect. The * relative conclusion (D1 dominates {@code update}, is roughly comparable on {@code add}, * ties on {@code iterate}) is unchanged either way. + * + *

      Separately rerun on Zulu 17.0.7 (native AArch64, same machine, pollution wiring unchanged; JMH + * auto-detected the cheap "compiler" Blackhole mode here, unlike JDK 8, so absolute numbers below + * are not comparable to the JDK 8 tables above — see {@code HashtableD2Benchmark}'s javadoc for the + * full caveat). M ops/us, 8 threads: + * + * 

      {@code
+ * add_hashMap        1502.6   add_hashtable      1377.3
+ * update_hashMap      644.2   update_hashtable   2706.5
+ * iterate_hashMap      19.3   iterate_hashtable    78.0
+ * }
      + * + *

      Within this single run (so the cross-JDK Blackhole-mode confound doesn't apply to the ratios), + * {@code update_hashtable} still wins by ~4.2x — down from ~14x on JDK 8, because Java 17's + * allocator/GC absorbs {@code update_hashMap}'s per-call {@code Long} boxing far better than JDK 8 + * did (update_hashMap itself got ~5x faster; update_hashtable only ~1.5x faster). {@code + * iterate_hashtable} also now clearly wins (~4.0x), flipping from JDK 8's "wash" — HashMap's {@code + * entrySet()} iterator does more per-entry work than a modern JIT's allocation improvements erase. + * {@code add} is the one case that flips the other way: {@code add_hashMap} edges out {@code + * add_hashtable} slightly (1502.6 vs 1377.3). Net takeaway: {@code Hashtable} is a strong + * substitute for {@code HashMap} particularly for simple counter/tally use cases with a primitive + * value, where avoiding the per-update boxing allocation pays off even on a JVM with much better + * allocation handling than JDK 8 had. */ @Fork(2) @Warmup(iterations = 2) From 7f6889d388d465d42607ac4e2e33318de5f84ff4 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 26 Aug 2026 09:26:32 -0400 Subject: [PATCH 21/52] Record Java 17 rerun for HashtableD2Benchmark Same controlled rerun as HashtableD1Benchmark. update_hashtable wins ~11.6x (down from ~26x on JDK 8); unlike JDK 8, Hashtable now also wins clearly on add (~1.8x) and iterate (~3.4x, up from a wash) -- Java 17's allocator/GC narrows the update margin but doesn't flip any operation in HashMap's favor for D2. Same Blackhole-mode caveat on absolute numbers as D1; within-run ratios are what's trustworthy here. --- .../trace/util/HashtableD2Benchmark.java | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java index 7c9e412d479..49357ab9a17 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java @@ -42,10 +42,13 @@ *

      The D2 variants additionally pay for a composite-key wrapper allocation in the HashMap path * (Java has no built-in tuple-as-key) — D2 sidesteps it by taking both key parts directly. * - *

      Update is where Hashtable dominates: D2 is ~26x faster, because the HashMap path - * allocates per call (a {@code Long}, plus a {@code Key2}) and the resulting GC pressure throttles - * throughput under multiple threads. Add is ~3x faster for D2 (Hashtable sidesteps the - * {@code Key2} allocation). Iterate is essentially a wash — both are bucket walks. + *

      Update is where Hashtable dominates: D2 is ~26x faster on JDK 8 (see the Java 17 rerun + * below for a narrower but still decisive margin), because the HashMap path allocates per call (a + * {@code Long}, plus a {@code Key2}) and the resulting GC pressure throttles throughput under + * multiple threads. Like D1, this is the headline case for {@code Hashtable}: a simple + * counter/tally with a primitive value is exactly where HashMap's autoboxing tax bites hardest. + * Add is ~3x faster for D2 (Hashtable sidesteps the {@code Key2} allocation). Iterate + * is essentially a wash on JDK 8, though not on Java 17 (see below). * MacBook M1 8 threads (Java 8) * * Benchmark Mode Cnt Score Error Units @@ -72,6 +75,32 @@ * runs as not directly comparable on absolute numbers. The relative conclusion (D2 dominates * {@code update}, wins {@code add} by avoiding the {@code Key2} allocation, ties on {@code * iterate}) is unchanged either way. + * + *

      Separately rerun on Zulu 17.0.7 (native AArch64, same machine, pollution wiring unchanged). + * JMH auto-detected the cheap "compiler" Blackhole mode on Java 17 (its log explicitly warns that + * Blackhole-mode differences between JVMs can swing results significantly), which JDK 8 cannot use + * — so absolute numbers below are not comparable to the JDK 8 tables above; only within-run + * ratios are, since both benchmark methods in a given run get identical Blackhole treatment. M + * ops/us, 8 threads: + * + * 

      {@code
+ * add_hashMap         656.7   add_hashtable     1185.5
+ * update_hashMap      196.7   update_hashtable  2292.2
+ * iterate_hashMap      20.5   iterate_hashtable   69.2
+ * }
      + * + *

      {@code update_hashtable} still wins decisively (~11.6x, down from ~26x on JDK 8 — Java 17's + * allocator/GC absorbs {@code update_hashMap}'s per-call {@code Long}+{@code Key2} boxing far + * better than JDK 8 did: update_hashMap got ~3.5x faster, update_hashtable only ~1.6x faster). + * Unlike JDK 8, Hashtable now wins clearly on every operation: {@code add_hashtable} wins + * ~1.8x (vs. JDK 8's ~3x — HashMap's {@code Key2} allocation also got relatively cheaper), and + * {@code iterate_hashtable} flips from JDK 8's wash to a ~3.4x win (HashMap's {@code entrySet()} + * iterator does more per-entry work than a modern JIT's allocation improvements erase). Net + * takeaway, consistent with {@link HashtableD1Benchmark}: {@code Hashtable} is a strong substitute + * for {@code HashMap} particularly for simple counter/tally use cases with a primitive value, where + * avoiding the per-update boxing allocation pays off even on a JVM with much better allocation + * handling than JDK 8 had — and for D2 specifically, avoiding the composite-key wrapper allocation + * pays off across the board, not just on {@code update}. */ @Fork(2) @Warmup(iterations = 2) From 3d9b93acadc9f6acb31d93600928752a0e33ae6b Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 22 Sep 2026 16:11:42 -0400 Subject: [PATCH 22/52] Address bot review findings on benchmark pollution helper and javadoc BenchmarkUtils's contains()/get() probes reused the exact same key instance just added/put, so HashMap/ConcurrentHashMap's internal identity fast path (key == storedKey) short-circuited before equals() was ever invoked -- the helper was polluting hashCode() dispatch but not equals() dispatch, undermining its own purpose (Codex P1, Autotest P2). Added distinctEqualCopy() to probe with an equal-but-not-identical instance instead. Also fixes: mislabeled "M ops/us" throughput unit in the Java 17 tables (HashtableD1/D2Benchmark, should read ops/us like the JDK 8 tables above them), a Javadoc comment in TagMapAccessBenchmark left describing ReadMap after a @Setup method was inserted above it, and an invalid cross-JDK comparison in SingleThreadedSetBenchmark's commentary that attributed unchanged numbers to "pollution had no effect" without acknowledging the JDK 8-vs-17 confound. Note: the BenchmarkUtils fix changes what polluteHashDispatch() actually exercises, so the "Java 17 rerun" numbers recorded in ThreadSafeMapD1/D2/ CounterBenchmark's javadoc (added in an earlier commit on this branch) are still pending a fresh rerun against this corrected helper. --- .../trace/api/TagMapAccessBenchmark.java | 8 ++--- .../datadog/trace/util/BenchmarkUtils.java | 35 ++++++++++++++++--- .../trace/util/HashtableD1Benchmark.java | 2 +- .../trace/util/HashtableD2Benchmark.java | 2 +- .../util/SingleThreadedSetBenchmark.java | 13 ++++--- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java index 3cd021a65b4..c8ab5a2367c 100644 --- a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java @@ -121,15 +121,15 @@ public class TagMapAccessBenchmark { } } - /** - * Pre-populated read map, PER-THREAD ({@code Scope.Thread}): each thread owns its own map so - * reads don't contend on shared mutable state under {@code @Threads(8)}. - */ @Setup(Level.Trial) public void setUp() { BenchmarkUtils.polluteHashDispatch(); } + /** + * Pre-populated read map, PER-THREAD ({@code Scope.Thread}): each thread owns its own map so + * reads don't contend on shared mutable state under {@code @Threads(8)}. + */ @State(Scope.Thread) public static class ReadMap { TagMap map; diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java index 7e2fd2d2ec2..e5ac9f2b0c5 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java +++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java @@ -75,7 +75,7 @@ public static void populateTypeProfile(Collection populated) { public static void populateTypeProfile(Collection populated, Object... decoyKeys) { for (Object key : decoyKeys) { - populated.contains(key); + populated.contains(distinctEqualCopy(key)); } } @@ -83,7 +83,7 @@ public static void populateTypeProfile(Collection populated, Object... d public static void populateTypeProfileMutable(Collection scratch, Object... decoyKeys) { for (Object key : decoyKeys) { scratch.add(key); - scratch.contains(key); + scratch.contains(distinctEqualCopy(key)); } } @@ -98,7 +98,7 @@ public static void populateTypeProfileMap(Map populated) { public static void populateTypeProfileMap(Map populated, Object... decoyKeys) { for (Object key : decoyKeys) { - populated.get(key); + populated.get(distinctEqualCopy(key)); } } @@ -110,7 +110,34 @@ public static void populateTypeProfileMutableMap( Map scratch, Object... decoyKeys) { for (Object key : decoyKeys) { scratch.put(key, key); - scratch.get(key); + scratch.get(distinctEqualCopy(key)); + } + } + + /** + * Returns a distinct instance that's {@code .equals()} to {@code key} but never {@code ==} it, so + * the lookup that follows can't take {@code HashMap}/{@code ConcurrentHashMap}'s internal {@code + * key == storedKey || key.equals(storedKey)} identity fast path and skip calling {@code equals()} + * -- which is exactly the dispatch this class exists to pollute. {@code Object}'s own {@code + * equals()} is identity, so a decoy of that type has no distinct-but-equal instance to make; it's + * returned as-is, and the identity fast path is then indistinguishable from a genuine {@code + * equals()} call anyway. + */ + @SuppressWarnings( + "deprecation") // boxed-type constructors: only way to force a non-cached instance + private static Object distinctEqualCopy(Object key) { + if (key instanceof String) { + return new String((String) key); + } else if (key instanceof Integer) { + return new Integer((Integer) key); + } else if (key instanceof Long) { + return new Long((Long) key); + } else if (key instanceof Double) { + return new Double((Double) key); + } else if (key instanceof Boolean) { + return new Boolean((Boolean) key); + } else { + return key; } } } diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java index 9581a8db520..d16ee3c4eca 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java @@ -77,7 +77,7 @@ *

      Separately rerun on Zulu 17.0.7 (native AArch64, same machine, pollution wiring unchanged; JMH * auto-detected the cheap "compiler" Blackhole mode here, unlike JDK 8, so absolute numbers below * are not comparable to the JDK 8 tables above — see {@code HashtableD2Benchmark}'s javadoc for the - * full caveat). M ops/us, 8 threads: + * full caveat). ops/us, 8 threads: * * 

      {@code
  * add_hashMap        1502.6   add_hashtable      1377.3
diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
index 49357ab9a17..1a181bee77c 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
@@ -80,7 +80,7 @@
  * JMH auto-detected the cheap "compiler" Blackhole mode on Java 17 (its log explicitly warns that
  * Blackhole-mode differences between JVMs can swing results significantly), which JDK 8 cannot use
  * — so absolute numbers below are not comparable to the JDK 8 tables above; only within-run
- * ratios are, since both benchmark methods in a given run get identical Blackhole treatment. M
+ * ratios are, since both benchmark methods in a given run get identical Blackhole treatment.
  * ops/us, 8 threads:
  *
  * 
      {@code
diff --git a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java
index 2b19e3ac48f..b498f10bc5f 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java
@@ -70,11 +70,14 @@
  *       {@code contains} is ~48% slower synchronized (1421 → 746M ops/s on JDK 8, vs. ~37% on Java
  *       17) — same story, somewhat larger tax. {@code iterate}'s tax stays small either way (~4%
  *       here): one monitor acquire amortized over the walk.
- *   
    • Type-profile pollution didn't change the qualitative story from the original Java 17 run —
- *       {@code contains_hashSet} and {@code iterate_hashSet} land in the same range (1291 vs 1421M,
- *       91 vs 134M) rather than collapsing, unlike {@link ImmutableSetBenchmark}'s {@code hitFresh}
- *       case. Construction numbers remain the noisiest (several {@code @Fork(2)} error bars exceed
- *       half the mean); {@code TreeSet} stays the slowest to build across both runs.
+ *   
    • {@code contains_hashSet} and {@code iterate_hashSet} land in the same range as the original
+ *       Java 17 run (1291 vs 1421M, 91 vs 134M) rather than collapsing, unlike {@link
+ *       ImmutableSetBenchmark}'s {@code hitFresh} case -- but that run is also on a different JDK,
+ *       so it isn't a clean pollution-only comparison (same caveat as {@link
+ *       HashtableD1Benchmark}'s Javadoc); read it as weak, not decisive, evidence that pollution
+ *       didn't change the qualitative story. Construction numbers remain the noisiest (several
+ *       {@code @Fork(2)} error bars exceed half the mean); {@code TreeSet} stays the slowest to
+ *       build across both runs.
  * 
  */
 @Fork(2)

From 56a31d96c558a8d85755afd153a01679c0b68e59 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Tue, 22 Sep 2026 17:02:48 -0400
Subject: [PATCH 23/52] Record JDK 8 rerun results with fixed pollution helper
 for ThreadSafeMap benchmarks

Reruns ThreadSafeMapD1/D2/CounterBenchmark with BenchmarkUtils.polluteHashDispatch()
now fixed (commit 3d9b93acad) to avoid the identity fast-path bypass. Documents the
fresh JDK 8 numbers alongside the pre-existing Java 17 tables, with the same
cross-JDK-comparison caveat used elsewhere in this PR since these reruns aren't on
the same JDK as the original tables.

Co-Authored-By: Claude Sonnet 5 
---
 .../util/ThreadSafeMapCounterBenchmark.java   | 18 ++++++++++++
 .../trace/util/ThreadSafeMapD1Benchmark.java  | 24 ++++++++++++++++
 .../trace/util/ThreadSafeMapD2Benchmark.java  | 28 +++++++++++++++++++
 3 files changed, 70 insertions(+)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java
index ac5cdbe3bd5..0d5ee26eecf 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java
@@ -53,6 +53,24 @@
  *       embedding the counter directly in the entry — one object instead of two, with no throughput
  *       penalty.
  * 
+ *
+ * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} wired into {@code SharedState.setUp()}
+ * (JDK 8 on this machine; the Java 17 table above predates pollution entirely, so this is also a
+ * cross-JDK comparison -- not a clean pollution-only delta, same caveat as {@link
+ * HashtableD1Benchmark}'s Javadoc):
+ *
+ * 
      {@code
+ * Benchmark                          Score   Units
+ * increment_longAdder                   87   ops/us
+ * increment_concurrentHashtable         71   ops/us
+ * increment_atomicLong                  70   ops/us
+ * }
      
+ *
+ * 
      {@code ConcurrentHashtable} and {@code AtomicLong} are still within 2% of each other (71 vs 70
+ * ops/us), unchanged from above. {@code LongAdder}'s lead widened (87 vs 71/70, vs. ~11% above),
+ * but its error bar here is larger than its own mean ({@code ±103} on a score of {@code 87}) --
+ * pure noise at this fork count, not a real widening; take the "within 15%" finding above as still
+ * the reliable read.
  */
 @Fork(2)
 @Warmup(iterations = 2)
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
index bf51916eef6..cac36918c10 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
@@ -63,6 +63,30 @@
  *   
    • {@code getOrCreate} is near-identical to {@code get} because all keys are pre-populated —
  *       the lock branch is never taken during measurement.
  * 
+ *
+ * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} wired into {@code SharedState.setUp()}
+ * (JDK 8 on this machine; the Java 17 table above predates pollution entirely, so this is also a
+ * cross-JDK comparison -- not a clean pollution-only delta, same caveat as {@link
+ * HashtableD1Benchmark}'s Javadoc):
+ *
+ * 
      {@code
+ * Benchmark                             Score   Units
+ * get_concurrentHashtable               1446   ops/us
+ * get_concurrentHashMap                 1161   ops/us
+ * get_concurrentSkipListMap              156   ops/us
+ * get_synchronizedHashMap                 30   ops/us
+ *
+ * getOrCreate_concurrentHashtable       1434   ops/us
+ * getOrCreate_concurrentHashMap         1139   ops/us
+ * getOrCreate_synchronizedHashMap        30   ops/us
+ * }
      
+ *
+ * 
      All four relative conclusions above still hold: {@code ConcurrentHashtable} still leads {@code
+ * ConcurrentHashMap} on {@code get} (~25%, down from ~38% -- within the JDK/pollution confound
+ * above, not necessarily a pollution effect on its own), {@code ConcurrentSkipListMap} and
+ * synchronized {@code HashMap} remain far behind, and {@code getOrCreate} still tracks {@code get}
+ * closely. {@code synchronizedHashMap}'s error bars are wide relative to its mean at
+ * {@code @Fork(2)} here -- directional only, not decisive.
  */
 @Fork(2)
 @Warmup(iterations = 2)
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
index 06e079086cb..daaf752577c 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
@@ -71,6 +71,34 @@
  *       traversal; the two-traversal {@code getOrCreate} pattern adds further overhead on misses.
  *   
    • Synchronized {@code HashMap} is ~50× slower than {@code ConcurrentHashtable}.
  * 
+ *
+ * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} wired into {@code SharedState.setUp()}
+ * (JDK 8 on this machine; the Java 17 table above predates pollution entirely, so this is also a
+ * cross-JDK comparison -- not a clean pollution-only delta, same caveat as {@link
+ * HashtableD1Benchmark}'s Javadoc):
+ *
+ * 
      {@code
+ * Benchmark                              Score   Units
+ * get_support                            1495   ops/us
+ * get_concurrentHashtable                1390   ops/us
+ * get_concurrentHashMap                   971   ops/us
+ * get_concurrentSkipListMap               138   ops/us
+ * get_synchronizedHashMap                  30   ops/us
+ *
+ * getOrCreate_support                    1408   ops/us
+ * getOrCreate_concurrentHashtable        1244   ops/us
+ * getOrCreate_concurrentHashMap           935   ops/us
+ * getOrCreate_concurrentSkipListMap       158   ops/us
+ * getOrCreate_synchronizedHashMap          30   ops/us
+ * }
      
+ *
+ * 
      {@code Support} and {@code ConcurrentHashtable} remain neck-and-neck on {@code get} (1495 vs
+ * 1390, {@code Support} now narrowly ahead rather than narrowly behind), both still clearly ahead
+ * of {@code ConcurrentHashMap} (~1.4-1.5x here, down from ~2x above) -- again within the
+ * JDK/pollution confound, not necessarily a pollution effect on its own. {@code
+ * ConcurrentSkipListMap} and synchronized {@code HashMap} remain far behind on both operations.
+ * {@code synchronizedHashMap}'s and {@code getOrCreate_concurrentHashtable}'s error bars are wide
+ * relative to their means at {@code @Fork(2)} here -- directional only, not decisive.
  */
 @Fork(2)
 @Warmup(iterations = 2)

From acd9bd09353620b973dacc44c7416c8959821f05 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Tue, 22 Sep 2026 18:27:52 -0400
Subject: [PATCH 24/52] Replace JDK 8 rerun with clean same-JDK pollution
 comparison

Reruns ThreadSafeMapD1/D2/CounterBenchmark on the same Zulu 17 JVM as the
original baseline table, superseding the earlier JDK 8 rerun (56a31d96c5)
whose numbers were confounded by a JDK 8 vs 17 optimizer difference on
arm64. The clean delta surfaces a real finding the confounded run
couldn't: synchronized HashMap throughput collapses by ~67-73% once hash
dispatch is polluted, far more than the lock contention these benchmarks
were designed to isolate.

Co-Authored-By: Claude Sonnet 5 
---
 .../util/ThreadSafeMapCounterBenchmark.java   | 22 +++++-----
 .../trace/util/ThreadSafeMapD1Benchmark.java  | 34 +++++++-------
 .../trace/util/ThreadSafeMapD2Benchmark.java  | 44 ++++++++++---------
 3 files changed, 50 insertions(+), 50 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java
index 0d5ee26eecf..15c131dbd62 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java
@@ -54,23 +54,21 @@
  *       penalty.
  * 
  *
- * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} wired into {@code SharedState.setUp()}
- * (JDK 8 on this machine; the Java 17 table above predates pollution entirely, so this is also a
- * cross-JDK comparison -- not a clean pollution-only delta, same caveat as {@link
- * HashtableD1Benchmark}'s Javadoc):
+ * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} wired into {@code
+ * SharedState.setUp()}, same JDK 17 as the table above -- a clean pollution-only delta:
  *
  * 
      {@code
  * Benchmark                          Score   Units
- * increment_longAdder                   87   ops/us
- * increment_concurrentHashtable         71   ops/us
- * increment_atomicLong                  70   ops/us
+ * increment_longAdder                  205   ops/us
+ * increment_atomicLong                  72   ops/us
+ * increment_concurrentHashtable         68   ops/us
  * }
      
  *
- * 
      {@code ConcurrentHashtable} and {@code AtomicLong} are still within 2% of each other (71 vs 70
- * ops/us), unchanged from above. {@code LongAdder}'s lead widened (87 vs 71/70, vs. ~11% above),
- * but its error bar here is larger than its own mean ({@code ±103} on a score of {@code 87}) --
- * pure noise at this fork count, not a real widening; take the "within 15%" finding above as still
- * the reliable read.
+ * 
      {@code ConcurrentHashtable} and {@code AtomicLong} are still within 6% of each other (68 vs 72
+ * ops/us), unchanged from above. {@code LongAdder}'s score jumped to 205 ops/us, but its error bar
+ * ({@code ±429}) is more than double its own mean -- unusable at this fork count, not evidence of a
+ * real pollution effect; take the "within 15%" finding above as still the reliable read for {@code
+ * LongAdder} too.
  */
 @Fork(2)
 @Warmup(iterations = 2)
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
index cac36918c10..5e1f6db6dd3 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
@@ -64,29 +64,29 @@
  *       the lock branch is never taken during measurement.
  * 
  *
- * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} wired into {@code SharedState.setUp()}
- * (JDK 8 on this machine; the Java 17 table above predates pollution entirely, so this is also a
- * cross-JDK comparison -- not a clean pollution-only delta, same caveat as {@link
- * HashtableD1Benchmark}'s Javadoc):
+ * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} wired into {@code
+ * SharedState.setUp()}, same JDK 17 as the table above -- a clean pollution-only delta:
  *
  * 
      {@code
  * Benchmark                             Score   Units
- * get_concurrentHashtable               1446   ops/us
- * get_concurrentHashMap                 1161   ops/us
- * get_concurrentSkipListMap              156   ops/us
- * get_synchronizedHashMap                 30   ops/us
+ * get_concurrentHashtable               1478   ops/us
+ * get_concurrentHashMap                 1188   ops/us
+ * get_concurrentSkipListMap              207   ops/us
+ * get_synchronizedHashMap                  9   ops/us
  *
- * getOrCreate_concurrentHashtable       1434   ops/us
- * getOrCreate_concurrentHashMap         1139   ops/us
- * getOrCreate_synchronizedHashMap        30   ops/us
+ * getOrCreate_concurrentHashtable       1549   ops/us
+ * getOrCreate_concurrentHashMap         1188   ops/us
+ * getOrCreate_synchronizedHashMap          9   ops/us
  * }
      
  *
- * 
      All four relative conclusions above still hold: {@code ConcurrentHashtable} still leads {@code
- * ConcurrentHashMap} on {@code get} (~25%, down from ~38% -- within the JDK/pollution confound
- * above, not necessarily a pollution effect on its own), {@code ConcurrentSkipListMap} and
- * synchronized {@code HashMap} remain far behind, and {@code getOrCreate} still tracks {@code get}
- * closely. {@code synchronizedHashMap}'s error bars are wide relative to its mean at
- * {@code @Fork(2)} here -- directional only, not decisive.
+ * 
      Synchronized {@code HashMap} collapses by ~73% (33/31 to 9 ops/us on {@code get}/{@code
+ * getOrCreate}) -- pollution turns its megamorphic {@code hashCode()}/{@code equals()} dispatch
+ * into most of its cost, far more than the lock contention this benchmark was designed to isolate.
+ * {@code ConcurrentHashtable} and {@code ConcurrentHashMap} both hold roughly steady (within ~7%),
+ * so the {@code ConcurrentHashtable} lead over {@code ConcurrentHashMap} narrows only slightly
+ * (~24-30%, down from ~38%) -- this table's own low error bars (all under 2% of their means) make
+ * that narrowing a real, if modest, effect rather than noise. {@code ConcurrentSkipListMap} rises
+ * slightly (~22%) but with an error bar spanning ~21% of its mean -- directional, not decisive.
  */
 @Fork(2)
 @Warmup(iterations = 2)
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
index daaf752577c..8b4e2299a81 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
@@ -72,33 +72,35 @@
  *   
    • Synchronized {@code HashMap} is ~50× slower than {@code ConcurrentHashtable}.
  * 
  *
- * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} wired into {@code SharedState.setUp()}
- * (JDK 8 on this machine; the Java 17 table above predates pollution entirely, so this is also a
- * cross-JDK comparison -- not a clean pollution-only delta, same caveat as {@link
- * HashtableD1Benchmark}'s Javadoc):
+ * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} wired into {@code
+ * SharedState.setUp()}, same JDK 17 as the table above -- a clean pollution-only delta:
  *
  * 
      {@code
  * Benchmark                              Score   Units
- * get_support                            1495   ops/us
- * get_concurrentHashtable                1390   ops/us
- * get_concurrentHashMap                   971   ops/us
- * get_concurrentSkipListMap               138   ops/us
- * get_synchronizedHashMap                  30   ops/us
+ * get_concurrentHashtable                1505   ops/us
+ * get_support                            1313   ops/us
+ * get_concurrentHashMap                  1072   ops/us
+ * get_concurrentSkipListMap               172   ops/us
+ * get_synchronizedHashMap                   9   ops/us
  *
- * getOrCreate_support                    1408   ops/us
- * getOrCreate_concurrentHashtable        1244   ops/us
- * getOrCreate_concurrentHashMap           935   ops/us
- * getOrCreate_concurrentSkipListMap       158   ops/us
- * getOrCreate_synchronizedHashMap          30   ops/us
+ * getOrCreate_support                    1516   ops/us
+ * getOrCreate_concurrentHashtable        1300   ops/us
+ * getOrCreate_concurrentHashMap          1108   ops/us
+ * getOrCreate_concurrentSkipListMap       178   ops/us
+ * getOrCreate_synchronizedHashMap           9   ops/us
  * }
      
  *
- * 
      {@code Support} and {@code ConcurrentHashtable} remain neck-and-neck on {@code get} (1495 vs
- * 1390, {@code Support} now narrowly ahead rather than narrowly behind), both still clearly ahead
- * of {@code ConcurrentHashMap} (~1.4-1.5x here, down from ~2x above) -- again within the
- * JDK/pollution confound, not necessarily a pollution effect on its own. {@code
- * ConcurrentSkipListMap} and synchronized {@code HashMap} remain far behind on both operations.
- * {@code synchronizedHashMap}'s and {@code getOrCreate_concurrentHashtable}'s error bars are wide
- * relative to their means at {@code @Fork(2)} here -- directional only, not decisive.
+ * 
      Synchronized {@code HashMap} collapses by ~67% (30/28 to 9 ops/us), the same magnitude seen in
+ * {@link ThreadSafeMapD1Benchmark} -- pollution dominates its cost far more than lock contention.
+ * {@code ConcurrentHashMap} rises ~38-44% over the unpolluted table (971→1072, 935→1108) with tight
+ * error bars (under 8% of the mean) -- a real effect, not noise; the {@link Key2} allocation this
+ * benchmark forces on every {@code ConcurrentHashMap} lookup apparently costs relatively less once
+ * the JIT already treats the surrounding dispatch as megamorphic. {@code Support} and {@code
+ * ConcurrentHashtable} are still the two fastest and remain close to each other, but neither can be
+ * called ahead here: their error bars are enormous (±520 and ±591, roughly 35-45% of their own
+ * means) and their relative order flips between {@code get} and {@code getOrCreate} -- read both as
+ * noise, not as a real reordering. {@code ConcurrentSkipListMap} rises modestly, also within a wide
+ * error bar -- directional only.
  */
 @Fork(2)
 @Warmup(iterations = 2)

From ecbdd4ffd1d3563b12be85235dd5bd22fcab464f Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Wed, 23 Sep 2026 09:03:20 -0400
Subject: [PATCH 25/52] Pollute compareTo and MapN dispatch in BenchmarkUtils

polluteHashDispatch() only made hash-based (equals()/hashCode())
dispatch megamorphic, leaving TreeMap/TreeSet/ConcurrentSkipListMap's
compareTo dispatch monomorphic and the JDK's immutable Map.copyOf
(MapN) lookup site untouched -- both flagged by review as able to
artificially favor those structures in benchmark comparisons.

Adds polluteCompareToDispatch(), driven from polluteHashDispatch()
since every current caller wants both passes: each decoy key type
gets its own scratch TreeSet/TreeMap/ConcurrentSkipListMap (natural
ordering can't mix incomparable types in one instance, but the JVM's
call-site type profile is shared across instances regardless). Also
exercises a Map.copyOf-backed MapN with a get() pass, matching the
existing Set.copyOf-backed SetN handling.

Co-Authored-By: Claude Sonnet 5 
---
 .../datadog/trace/util/BenchmarkUtils.java    | 65 ++++++++++++++++++-
 1 file changed, 62 insertions(+), 3 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index e5ac9f2b0c5..d61d5292c15 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -2,9 +2,13 @@
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListMap;
 
 /** Shared setup helpers for JMH benchmarks in this module. */
 public final class BenchmarkUtils {
@@ -33,9 +37,14 @@ private BenchmarkUtils() {}
    * same internal dispatch call site. {@code ConcurrentHashMap} does not: it's an unrelated class
    * with its own {@code hashCode()}/{@code equals()} call sites, so it needs its own scratch
    * instance (also covers {@code ConcurrentHashMap#newKeySet()}, which is backed by a {@code
-   * ConcurrentHashMap}). Structures that dispatch on {@code compareTo} instead ({@code TreeMap},
-   * {@code TreeSet}, {@code ConcurrentSkipListMap}) aren't affected by any of this and don't need
-   * pollution.
+   * ConcurrentHashMap}). The JDK's immutable {@code Set.copyOf}/{@code Map.copyOf} ({@code SetN}/
+   * {@code MapN}) have their own internal {@code equals()} call sites too, distinct from {@code
+   * HashSet}/{@code HashMap}'s, so they get their own scratch instances as well.
+   *
+   * 
      {@code TreeMap}/{@code TreeSet}/{@code ConcurrentSkipListMap} dispatch on {@code compareTo}
+   * instead of {@code hashCode()}/{@code equals()}, so they need a separate pass -- see {@link
+   * #polluteCompareToDispatch()}, which this method also drives, since every caller of this method
+   * wants both passes.
    *
    * 
      Deliberately does not touch the benchmark's own {@code contains}/{@code add}/{@code get}
    * call sites -- those are realistically free to specialize per caller, the way a genuinely hot,
@@ -59,6 +68,56 @@ public static void polluteHashDispatch(Object... decoyKeys) {
     populateTypeProfileMutable(new HashSet<>(), decoyKeys);
     populateTypeProfile(CollectionUtils.tryMakeImmutableSet(Arrays.asList(decoyKeys)), decoyKeys);
     populateTypeProfileMutableMap(new ConcurrentHashMap<>(), decoyKeys);
+
+    Map mapCopySource = new HashMap<>();
+    for (Object key : decoyKeys) {
+      mapCopySource.put(key, key);
+    }
+    populateTypeProfileMap(CollectionUtils.tryMakeImmutableMap(mapCopySource), decoyKeys);
+
+    polluteCompareToDispatch(decoyKeys);
+  }
+
+  /**
+   * Counterpart to the rest of this class for the {@code compareTo}-based dispatch used by {@code
+   * TreeMap}/{@code TreeSet}/{@code ConcurrentSkipListMap}, instead of {@code hashCode()}/{@code
+   * equals()}.
+   *
+   * 
      Unlike the hash-based structures above, a single sorted collection can't hold multiple decoy
+   * key classes at once: natural ordering calls {@code key.compareTo(existing)}, which throws
+   * {@code ClassCastException} the moment two mutually-incomparable types meet (e.g. a {@code
+   * String} and an {@code Integer}). So each key type gets its own scratch instance here, one type
+   * at a time. That's still enough to make the dispatch megamorphic: HotSpot's type profile lives
+   * on the bytecode call site inside {@code TreeMap}/{@code TreeSet}/{@code
+   * ConcurrentSkipListMap}'s shared implementation, not on any one collection instance, so driving
+   * several receiver types through that site across several scratch instances pollutes it exactly
+   * as effectively as driving them through one shared instance would.
+   *
+   * 
      {@code Object}'s decoy is skipped: it isn't {@link Comparable}, so it has no natural
+   * ordering to dispatch through in the first place -- the same reason it's exempt from an {@code
+   * equals()}-identity copy in {@link #distinctEqualCopy}.
+   */
+  public static void polluteCompareToDispatch() {
+    polluteCompareToDispatch(DEFAULT_DECOY_KEYS);
+  }
+
+  public static void polluteCompareToDispatch(Object... decoyKeys) {
+    for (Object key : decoyKeys) {
+      if (!(key instanceof Comparable)) {
+        continue;
+      }
+      TreeSet treeSet = new TreeSet<>();
+      treeSet.add(key);
+      treeSet.contains(distinctEqualCopy(key));
+
+      TreeMap treeMap = new TreeMap<>();
+      treeMap.put(key, key);
+      treeMap.get(distinctEqualCopy(key));
+
+      ConcurrentSkipListMap skipListMap = new ConcurrentSkipListMap<>();
+      skipListMap.put(key, key);
+      skipListMap.get(distinctEqualCopy(key));
+    }
   }
 
   /**

From 2c0117e358bf199d4417aa0c6a633e6ab5234fab Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Wed, 23 Sep 2026 09:39:10 -0400
Subject: [PATCH 26/52] Add a perf-review rubric check for Map/Set JMH
 benchmark dispatch pollution

Add J12 (checks.md) and a matching guide.md callout: a benchmark that
compares Map/Set implementations using only one key class leaves the
structure's hashCode()/equals()/compareTo call site monomorphic,
overstating its throughput relative to production -- exactly what
PR #12298 found (synchronized HashMap collapsed ~70% once dispatch
was polluted). Points reviewers at BenchmarkUtils.polluteHashDispatch().

Co-Authored-By: Claude Sonnet 5 
---
 .agents/skills/perf-review/references/checks.md | 4 +++-
 .agents/skills/perf-review/references/guide.md  | 9 +++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/.agents/skills/perf-review/references/checks.md b/.agents/skills/perf-review/references/checks.md
index 05681822689..cd594f01549 100644
--- a/.agents/skills/perf-review/references/checks.md
+++ b/.agents/skills/perf-review/references/checks.md
@@ -51,7 +51,9 @@ Refines the universal checks with JVM mechanics. Quarantined here, for the Java
 - **J9 — `Objects.hash(...)` varargs / boxing hash on a hot path → `HashingUtils`** *(refines #1)*. The allocation is specific to the **varargs/boxing forms**: `Objects.hash(a, b, …)` allocates an `Object[]` per call and **boxes every primitive** arg; same for boxing primitives into a `new Object[]{…}` (or `Arrays.hashCode` over such an array). Per-span tag/key building, or a hot value object's `hashCode()` built this way, → a guaranteed per-call allocation + boxing. Fix: `datadog.trace.util.HashingUtils` — primitive `hash(long/int/boolean/char/…)` overloads (no boxing), `hash(Object,Object)` and `hash(int,int)` combiners (no array); for >2 fields fold pairwise through `hash(int,int)` (there is no varargs form, by design). flag-with-confidence for the varargs/boxing form — SEV-2/3. **Do NOT flag allocation-free combines** — a hand-rolled `31*h + Long.hashCode(x)` / `31*h + intField`, or `Arrays.hashCode` over an *existing primitive array*, allocates nothing (`HashingUtils` is itself 31-based); flagging them would recommend replacing already-correct code.
 - **J10 — hot-path `String.format` / string munging → `Strings` (+ `SubSequence`)** *(refines #2)*. `String.format` parses the format string, boxes its args, and allocates on every call — never on a hot path; hand-rolled case-conversion, class/resource-name munging, blank-checks, and truncation recomputed per call qualify too. Fix: `datadog.trace.util.Strings` — allocation-aware `replace`/`truncate(CharSequence)`/`isBlank`/`getResourceName`/`getClassName`/…; for **transient substring compares** prefer a `SubSequence` view (J7); for plain assembly, direct concatenation beats `format`. flag-with-confidence for `String.format` on a hot path; flag-as-measure for borderline munging — SEV-2/3.
 - **J11 — composite / multi-dimensional key maps on a hot path → `Hashtable` / `ConcurrentHashtable`** *(refines #1, #3)*. `Map>` nesting, or a `HashMap` keyed by a composite key (client-side stats, per-`(service, operation, …)` aggregation), allocates nested maps + `Entry` objects + boxes keys on the hot aggregation path. Fix: `datadog.trace.util.Hashtable` (single-threaded, composite-key D1/D2 tables — landed) or `datadog.trace.util.ConcurrentHashtable` (lock-free concurrent, **coming**) — positional composite keys, fewer allocations. flag-as-measure — SEV-2/3.
-- **Toolkit availability — cite only what exists.** Available today: `Strings`, `SubSequence`, `HashingUtils`, `Hashtable` (all `datadog.trace.util`), `RE2J` (`com.google.re2j`). Coming (name as "coming", don't imply it's present): `ConcurrentHashtable`, `StringIndex` (immutable string set/map), `UTF8BytesString.Cache` (recurring-string interner), wider `IntegerCache` (http-status/port boxing), `DDCache` inlining. **J7–J11 route an *existing* #1/#2/#3 finding to a reusable fix — they are not new flag-triggers. Don't raise a finding you wouldn't have raised anyway; the posture (precision, silent-when-unsure, findings-cap-scales-with-diff — see `SKILL.md`) is unchanged.**
+- **J12 — Benchmark validity: pollute Map/Set dispatch before trusting a JMH comparison** *(methodology check on a submitted benchmark, not a production hot-path finding — refines the flag-as-measure posture in J2)*. A JMH benchmark comparing `HashMap`/`HashSet`/`ConcurrentHashMap` (or `TreeMap`/`TreeSet`/`ConcurrentSkipListMap`, or the JDK's immutable `Map.of`/`Set.of` `MapN`/`SetN`) against another collection, that only ever looks up one key class for the whole run, leaves that collection's internal `hashCode()`/`equals()` (or `compareTo`) dispatch site artificially monomorphic — a state production rarely reaches, since those call sites are shared JVM-wide across every key type the whole process uses. An unpolluted benchmark can inflate the affected structure's throughput and flip or exaggerate a comparison (this is exactly what PR #12298 found and fixed: synchronized `HashMap` collapsed ~70% once its dispatch was made megamorphic). Before trusting such a benchmark's numbers, check its `@Setup` calls `datadog.trace.util.BenchmarkUtils.polluteHashDispatch()` (covers `HashSet`/`HashMap`/`ConcurrentHashMap`/`SetN`/`MapN` `equals()` dispatch and `TreeMap`/`TreeSet`/`ConcurrentSkipListMap` `compareTo` dispatch) before the measured operations run. **flag-with-confidence** when a new or edited Map/Set JMH benchmark in this module omits it — fix: call `BenchmarkUtils.polluteHashDispatch()` in `@Setup(Level.Trial)`/`Level.Iteration`, ahead of the benchmarked calls.
+
+- **Toolkit availability — cite only what exists.** Available today: `Strings`, `SubSequence`, `HashingUtils`, `Hashtable`, `BenchmarkUtils.polluteHashDispatch()`/`polluteCompareToDispatch()` for JMH setup (all `datadog.trace.util`), `RE2J` (`com.google.re2j`). Coming (name as "coming", don't imply it's present): `ConcurrentHashtable`, `StringIndex` (immutable string set/map), `UTF8BytesString.Cache` (recurring-string interner), wider `IntegerCache` (http-status/port boxing), `DDCache` inlining. **J7–J11 route an *existing* #1/#2/#3 finding to a reusable fix — they are not new flag-triggers. Don't raise a finding you wouldn't have raised anyway; the posture (precision, silent-when-unsure, findings-cap-scales-with-diff — see `SKILL.md`) is unchanged.**
 - **J5 — Cardinality-sensitive aggregator** *(domain-specialized #3)*. Some structures are invisible to the generic "unbounded collection" check because the risk is *cardinality*, not raw size: a config- or user-driven value (tag key, resource name, HTTP URL) feeding a **cardinality-sensitive aggregator** (e.g. the conflating metrics aggregator — each unique label combination = one aggregate; a `maxAggregates` cap bounds OOM but high-cardinality input *thrashes* it: constant eviction, garbled metrics). flag-with-confidence when config/user-driven values reach an aggregator with a per-key budget — **SEV-1** (same class as unbounded memory: correctness + heap impact). Fix: bound the source cardinality before it enters the aggregator, or use sentinel substitution for over-cap values. This surfaced on merged production code more than once in back-test calibration — the capstone pattern where the bot's value concentrates.
 
 ## Instrumentation (ByteBuddy Advice) idioms — dd-trace-java-specific fixes
diff --git a/.agents/skills/perf-review/references/guide.md b/.agents/skills/perf-review/references/guide.md
index 6f847ef39d0..2f77a78b22a 100644
--- a/.agents/skills/perf-review/references/guide.md
+++ b/.agents/skills/perf-review/references/guide.md
@@ -161,6 +161,15 @@ escape analysis eliminates *local* short-lived allocations, but only when the ob
 Stored in a map, returned, captured by a lambda, or passed to a non-inlined virtual call: it
 escapes, and it's real.
 
+**Map/Set JMH benchmarks that never pollute dispatch — treat as unverified.** A benchmark that
+looks up only one key class for its whole run leaves the collection's internal
+`hashCode()`/`equals()` (or `compareTo` for sorted maps/sets) call site artificially
+monomorphic — production hits that same shared call site with whatever key types the whole
+process uses, so it's realistically almost always megamorphic. An unpolluted benchmark can
+overstate a structure's throughput and flip a comparison (PR #12298: a synchronized `HashMap`
+collapsed ~70% once pollution was added). Check that the benchmark's `@Setup` calls
+`datadog.trace.util.BenchmarkUtils.polluteHashDispatch()` before trusting its numbers.
+
 **EA claims for scope/wrapper objects spanning I/O — treat as unverified.** A microbenchmark
 tight-loop can show zero allocation for a scope or wrapper object because C2 inlines through
 everything and scalar-replaces it. In production, scopes almost always wrap I/O — and C2 cannot

From 71a383ed5821fb8f384e2e46df402885e8154b22 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Wed, 23 Sep 2026 11:31:15 -0400
Subject: [PATCH 27/52] Re-pollute hash/compareTo dispatch every invocation,
 not once per trial

A one-shot BenchmarkUtils.polluteHashDispatch() call at @Setup(Level.Trial)
or Level.Iteration) gets drowned out by the benchmark's own real-key traffic
before HotSpot compiles the shared internal dispatch call sites, letting them
re-specialize to a dominant receiver. Confirmed via -XX:+PrintInlining on
ImmutableSetBenchmark.hashSet_hitFresh: HashMap.getNode's key.equals(k) call
showed "no static binding" (genuinely virtual) only once pollution moved to
@Setup(Level.Invocation), unlike the one-shot version.

Also fixes a transcription error in ThreadSafeMapD2Benchmark's pollution
delta prose: the unpolluted baseline quoted (971/935) didn't match this
file's own baseline table (777/769 ops/us).

Co-Authored-By: Claude Sonnet 5 
---
 .../trace/util/CaseInsensitiveMapBenchmark.java       |  8 ++++++--
 .../datadog/trace/util/ImmutableMapBenchmark.java     |  9 +++++++++
 .../datadog/trace/util/ImmutableSetBenchmark.java     |  9 +++++++++
 .../trace/util/SingleThreadedMapBenchmark.java        | 11 +++++++++--
 .../trace/util/SingleThreadedSetBenchmark.java        | 11 +++++++++--
 .../datadog/trace/util/ThreadSafeMapD1Benchmark.java  |  9 +++++++++
 .../datadog/trace/util/ThreadSafeMapD2Benchmark.java  | 11 ++++++++++-
 7 files changed, 61 insertions(+), 7 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
index 62a48976691..52f5485b537 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
@@ -128,8 +128,12 @@ static  T init(Supplier supplier) {
   // masking exactly the differences this benchmark compares.
   int lookupIndex = 0;
 
-  @Setup(Level.Trial)
-  public void setUp() {
+  // Re-pollute every invocation: a one-shot Level.Trial call gets drowned out by this
+  // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
+  // sites, letting them re-specialize to a dominant receiver (see
+  // BenchmarkUtils#polluteHashDispatch).
+  @Setup(Level.Invocation)
+  public void pollute() {
     BenchmarkUtils.polluteHashDispatch();
   }
 
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
index 7f21e08656f..dc2f7ad035a 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
@@ -161,6 +161,15 @@ public void setUp() {
   public static class Cursor {
     int index = 0;
 
+    // Re-pollute every invocation: a one-shot Level.Trial call gets drowned out by this
+    // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
+    // sites, letting them re-specialize to a dominant receiver (see
+    // BenchmarkUtils#polluteHashDispatch).
+    @Setup(Level.Invocation)
+    public void pollute() {
+      BenchmarkUtils.polluteHashDispatch();
+    }
+
     String nextKey() {
       return nextKey(EQUAL_KEYS);
     }
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java
index b0977dc5ecb..9b322e11af9 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java
@@ -164,6 +164,15 @@ public static class Cursor {
     int hitFreshIndex = 0;
     int missIndex = 0;
 
+    // Re-pollute every invocation: a one-shot Level.Trial call gets drowned out by this
+    // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
+    // sites, letting them re-specialize to a dominant receiver (see
+    // BenchmarkUtils#polluteHashDispatch).
+    @Setup(Level.Invocation)
+    public void pollute() {
+      BenchmarkUtils.polluteHashDispatch();
+    }
+
     String nextHit() {
       int i = hitIndex + 1;
       if (i >= STRINGS.length) {
diff --git a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java
index 26753867d18..36f824c5aad 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java
@@ -231,10 +231,17 @@ static IntEntry[] newFilledFlat() {
   IntEntry[] flatTable;
   int index = 0;
 
-  @Setup(Level.Trial)
-  public void setUp() {
+  // Re-pollute every invocation: a one-shot Level.Trial call gets drowned out by this
+  // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
+  // sites, letting them re-specialize to a dominant receiver (see
+  // BenchmarkUtils#polluteHashDispatch).
+  @Setup(Level.Invocation)
+  public void pollute() {
     BenchmarkUtils.polluteHashDispatch();
+  }
 
+  @Setup(Level.Trial)
+  public void setUp() {
     hashMap = new HashMap<>();
     fill(hashMap);
     synchronizedHashMap = Collections.synchronizedMap(new HashMap<>(hashMap));
diff --git a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java
index b498f10bc5f..04cb4c43034 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java
@@ -115,10 +115,17 @@ static void fill(Set set) {
   LinkedHashSet linkedHashSet;
   int index = 0;
 
-  @Setup(Level.Trial)
-  public void setUp() {
+  // Re-pollute every invocation: a one-shot Level.Trial call gets drowned out by this
+  // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
+  // sites, letting them re-specialize to a dominant receiver (see
+  // BenchmarkUtils#polluteHashDispatch).
+  @Setup(Level.Invocation)
+  public void pollute() {
     BenchmarkUtils.polluteHashDispatch();
+  }
 
+  @Setup(Level.Trial)
+  public void setUp() {
     hashSet = new HashSet<>(Arrays.asList(ELEMENTS));
     synchronizedSet = Collections.synchronizedSet(new HashSet<>(hashSet));
     treeSet = new TreeSet<>(Arrays.asList(ELEMENTS));
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
index 5e1f6db6dd3..6c63ea17249 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
@@ -147,6 +147,15 @@ public void setUp() {
   public static class ThreadState {
     int cursor;
 
+    // Re-pollute every invocation: a one-shot Level.Iteration call gets drowned out by this
+    // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
+    // sites, letting them re-specialize to a dominant receiver (see
+    // BenchmarkUtils#polluteHashDispatch).
+    @Setup(Level.Invocation)
+    public void pollute() {
+      BenchmarkUtils.polluteHashDispatch();
+    }
+
     int next() {
       int i = cursor;
       cursor = (i + 1) & (N_KEYS - 1);
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
index 8b4e2299a81..aa3f18d3222 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
@@ -92,7 +92,7 @@
  *
  * 
      Synchronized {@code HashMap} collapses by ~67% (30/28 to 9 ops/us), the same magnitude seen in
  * {@link ThreadSafeMapD1Benchmark} -- pollution dominates its cost far more than lock contention.
- * {@code ConcurrentHashMap} rises ~38-44% over the unpolluted table (971→1072, 935→1108) with tight
+ * {@code ConcurrentHashMap} rises ~38-44% over the unpolluted table (777→1072, 769→1108) with tight
  * error bars (under 8% of the mean) -- a real effect, not noise; the {@link Key2} allocation this
  * benchmark forces on every {@code ConcurrentHashMap} lookup apparently costs relatively less once
  * the JIT already treats the surrounding dispatch as megamorphic. {@code Support} and {@code
@@ -241,6 +241,15 @@ public void setUp() {
   public static class ThreadState {
     int cursor;
 
+    // Re-pollute every invocation: a one-shot Level.Iteration call gets drowned out by this
+    // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
+    // sites, letting them re-specialize to a dominant receiver (see
+    // BenchmarkUtils#polluteHashDispatch).
+    @Setup(Level.Invocation)
+    public void pollute() {
+      BenchmarkUtils.polluteHashDispatch();
+    }
+
     int next() {
       int i = cursor;
       cursor = (i + 1) & (N_KEYS - 1);

From 74435ace12a9b8ef7fad1fb370b80a259455ed9c Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Thu, 24 Sep 2026 22:09:06 -0400
Subject: [PATCH 28/52] Simplify pollution warm-up to a single scratch-free
 entry point

Replace the PollutionScratch/CompareToScratch persistent-scratch-object
machinery with a single warmUpHashDispatch(Blackhole) call: each pass now
allocates fresh collections and consumes the collection instances directly
via the Blackhole to defeat scalar replacement, instead of reusing
persistent fields. Removes the now-dead populateTypeProfile*/decoyKeys
varargs surface that had no callers, so using this in a new benchmark
needs nothing more than one @Setup(Level.Trial) method with no scratch
object to construct or wire up.

Co-Authored-By: Claude Sonnet 5 
---
 .../trace/api/TagMapAccessBenchmark.java      |   6 +-
 .../datadog/trace/util/BenchmarkUtils.java    | 216 +++++++++---------
 .../util/CaseInsensitiveMapBenchmark.java     |  86 +++----
 .../trace/util/HashtableD1Benchmark.java      |  18 +-
 .../trace/util/HashtableD2Benchmark.java      |  12 +-
 .../trace/util/ImmutableMapBenchmark.java     |  50 ++--
 .../trace/util/ImmutableSetBenchmark.java     |  63 +++--
 .../util/SingleThreadedMapBenchmark.java      |  65 +++---
 .../util/SingleThreadedSetBenchmark.java      |  73 ++----
 .../util/ThreadSafeMapCounterBenchmark.java   |  14 +-
 .../trace/util/ThreadSafeMapD1Benchmark.java  |  67 ++----
 .../trace/util/ThreadSafeMapD2Benchmark.java  |  91 +++-----
 12 files changed, 325 insertions(+), 436 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java
index c8ab5a2367c..377d528cbe6 100644
--- a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java
@@ -59,7 +59,7 @@
  * 
  *
  * 
      Rerun on JDK 8 with a new top-level {@code @Setup(Level.Trial)} calling {@link
- * BenchmarkUtils#polluteHashDispatch()} (this file had none before). M ops/s, 8 threads:
+ * BenchmarkUtils#warmUpHashDispatch} (this file had none before). M ops/s, 8 threads:
  *
  * 
      {@code
  * getEntry                        83   getObject                    87
@@ -122,8 +122,8 @@ public class TagMapAccessBenchmark {
   }
 
   @Setup(Level.Trial)
-  public void setUp() {
-    BenchmarkUtils.polluteHashDispatch();
+  public void setUp(Blackhole bh) {
+    BenchmarkUtils.warmUpHashDispatch(bh);
   }
 
   /**
diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index d61d5292c15..6b8cdf9e4b8 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -1,50 +1,55 @@
 package datadog.trace.util;
 
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentSkipListMap;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.infra.Blackhole;
 
 /** Shared setup helpers for JMH benchmarks in this module. */
 public final class BenchmarkUtils {
   private BenchmarkUtils() {}
 
-  private static final Object[] DEFAULT_DECOY_KEYS = {
-    "decoy", 1, 1L, 1.0d, Boolean.TRUE, new Object()
-  };
+  private static final Object[] DECOY_KEYS = {"decoy", 1, 1L, 1.0d, Boolean.TRUE, new Object()};
 
   /**
-   * Exercises {@link HashSet}/{@link java.util.HashMap}, the tracer's {@link
-   * CollectionUtils#tryMakeImmutableSet} immutable sets, and {@link ConcurrentHashMap} with several
-   * distinct key classes, so each structure's internal {@code hashCode()}/{@code equals()} dispatch
-   * -- a call site shared JVM-wide by every instance of that structure in the process, regardless
-   * of which specific instance or call site invokes {@code add}/{@code contains}/{@code get} -- is
+   * Number of pollution passes {@link #warmUpHashDispatch} runs before a trial starts. Driving
+   * enough calls at {@code Level.Trial} -- entirely before JMH's warmup, let alone measurement,
+   * starts -- gets HotSpot's tiered compiler through both C1 (tier 3, full profiling; default
+   * invocation threshold in the hundreds) and C2 (tier 4; default thresholds in the thousands) on
+   * the shared hash-dispatch call sites while decoy types are still part of their profile, so the
+   * resulting compiled code keeps a genuine multi-receiver-type guard for the rest of the trial --
+   * recompilation of already-C2-compiled code isn't triggered merely by one receiver type going
+   * quiet, only by hitting an actual uncommon trap or similar deopt event, so it doesn't need
+   * refreshing once compiled this way. This iteration count was picked by watching {@code
+   * -XX:+PrintCompilation}/{@code -XX:+PrintInlining} during development and confirming both tiers
+   * compile well before it's reached, with margin for ambient JVM-wide traffic on these shared JDK
+   * call sites; it isn't verified at runtime (there's no portable, non-test-only JDK API for "is
+   * this method at tier N").
+   */
+  private static final int WARM_UP_ITERATIONS = 50_000;
+
+  /**
+   * Call once from {@code @Setup(Level.Trial)}, passing the {@link Blackhole} JMH injects into the
+   * setup method. Exercises {@link HashSet}/{@link java.util.HashMap}, the tracer's {@link
+   * CollectionUtils#tryMakeImmutableSet} immutable sets, {@link ConcurrentHashMap}, and {@link
+   * TreeMap}/{@link TreeSet}/{@link ConcurrentSkipListMap} with several distinct key classes, so
+   * each structure's internal {@code hashCode()}/{@code equals()}/{@code compareTo()} dispatch -- a
+   * call site shared JVM-wide by every instance of that structure in the process, regardless of
+   * which specific instance or call site invokes {@code add}/{@code contains}/{@code get} -- is
    * already megamorphic before a benchmark measures lookups against a single key type.
    *
-   * 
      This matches production: those shared internal call sites are hit by every hash-based
-   * structure in the JVM across whatever key types the whole application uses, so they're
-   * realistically almost always megamorphic. An isolated benchmark that only ever looks up one key
-   * type (e.g. {@code String}) would otherwise leave them artificially monomorphic for the entire
-   * run, understating real dispatch cost.
-   *
-   * 
      {@code HashSet} is backed by {@code HashMap} in the JDK, so polluting it also covers plain
-   * {@code HashMap} and {@code LinkedHashMap} (which extends {@code HashMap}) -- they share the
-   * same internal dispatch call site. {@code ConcurrentHashMap} does not: it's an unrelated class
-   * with its own {@code hashCode()}/{@code equals()} call sites, so it needs its own scratch
-   * instance (also covers {@code ConcurrentHashMap#newKeySet()}, which is backed by a {@code
-   * ConcurrentHashMap}). The JDK's immutable {@code Set.copyOf}/{@code Map.copyOf} ({@code SetN}/
-   * {@code MapN}) have their own internal {@code equals()} call sites too, distinct from {@code
-   * HashSet}/{@code HashMap}'s, so they get their own scratch instances as well.
-   *
-   * 
      {@code TreeMap}/{@code TreeSet}/{@code ConcurrentSkipListMap} dispatch on {@code compareTo}
-   * instead of {@code hashCode()}/{@code equals()}, so they need a separate pass -- see {@link
-   * #polluteCompareToDispatch()}, which this method also drives, since every caller of this method
-   * wants both passes.
+   * 
      This matches production: those shared internal call sites are hit by every hash- or
+   * sorted-based structure in the JVM across whatever key types the whole application uses, so
+   * they're realistically almost always megamorphic. An isolated benchmark that only ever looks up
+   * one key type (e.g. {@code String}) would otherwise leave them artificially monomorphic for the
+   * entire run, understating real dispatch cost.
    *
    * 
      Deliberately does not touch the benchmark's own {@code contains}/{@code add}/{@code get}
    * call sites -- those are realistically free to specialize per caller, the way a genuinely hot,
@@ -54,122 +59,100 @@ private BenchmarkUtils() {}
    * ThreadSafeMapBenchmark} ({@code KeyStrategy} implementors referenced only so they're loaded,
    * never invoked): that technique denies class-hierarchy analysis a single-implementor bet for a
    * narrow, dd-trace-java-owned interface, and works by class-loading alone. It doesn't apply here
-   * -- {@code Object.hashCode()}/{@code equals()} already have countless implementors loaded in any
-   * real JVM, so a single-implementor CHA bet was never available for them. What gates their
-   * dispatch is the interpreter's per-call-site type profile, which only invocation can pollute --
-   * hence this helper actually calls {@code add}/{@code contains}/{@code get}, rather than just
-   * loading classes.
+   * -- {@code Object.hashCode()}/{@code equals()}/{@code compareTo()} already have countless
+   * implementors loaded in any real JVM, so a single-implementor CHA bet was never available for
+   * them. What gates their dispatch is the interpreter's per-call-site type profile, which only
+   * invocation can pollute -- hence this helper actually calls {@code add}/{@code contains}/{@code
+   * get}, rather than just loading classes.
+   *
+   * 
      {@code @Setup(Level.Trial)}, not {@code Level.Invocation}: the latter's cost is included in
+   * every {@code Throughput}/{@code AverageTime} measurement's own timed window (JMH has no way to
+   * subtract per-invocation setup cost without adding per-op {@code System.nanoTime()} overhead of
+   * its own), so once pollution isn't free relative to the benchmarked op, it would dominate the
+   * reported number instead of the thing being measured.
+   *
+   * 
      Every collection here is freshly allocated per pass and immediately consumed via {@code bh}
+   * -- not just the {@code boolean}/lookup results, but the collection instances themselves.
+   * Consuming only a lookup's result would leave the freshly-allocated, never-escaping collection
+   * open to scalar replacement: once HotSpot compiles this loop as its own unit and proves a
+   * just-allocated {@code HashSet} never escapes it, escape analysis can devirtualize the {@code
+   * hashCode()}/{@code equals()} calls against that specific instance directly, bypassing the
+   * shared, megamorphic call site entirely. Forcing every collection itself through the {@link
+   * Blackhole} closes that loophole, so the allocations here don't need to be reused across calls
+   * the way a measured hot path would.
    */
-  public static void polluteHashDispatch() {
-    polluteHashDispatch(DEFAULT_DECOY_KEYS);
+  public static void warmUpHashDispatch(Blackhole bh) {
+    for (int i = 0; i < WARM_UP_ITERATIONS; ++i) {
+      polluteHashDispatch(bh);
+      polluteCompareToDispatch(bh);
+    }
   }
 
-  public static void polluteHashDispatch(Object... decoyKeys) {
-    populateTypeProfileMutable(new HashSet<>(), decoyKeys);
-    populateTypeProfile(CollectionUtils.tryMakeImmutableSet(Arrays.asList(decoyKeys)), decoyKeys);
-    populateTypeProfileMutableMap(new ConcurrentHashMap<>(), decoyKeys);
-
+  private static void polluteHashDispatch(Blackhole bh) {
+    HashSet hashSet = new HashSet<>();
+    ConcurrentHashMap concurrentHashMap = new ConcurrentHashMap<>();
     Map mapCopySource = new HashMap<>();
-    for (Object key : decoyKeys) {
+    for (Object key : DECOY_KEYS) {
+      hashSet.add(distinctEqualCopy(key));
+      bh.consume(hashSet.contains(distinctEqualCopy(key)));
+
+      concurrentHashMap.put(distinctEqualCopy(key), key);
+      bh.consume(concurrentHashMap.get(distinctEqualCopy(key)));
+
       mapCopySource.put(key, key);
     }
-    populateTypeProfileMap(CollectionUtils.tryMakeImmutableMap(mapCopySource), decoyKeys);
-
-    polluteCompareToDispatch(decoyKeys);
+    bh.consume(hashSet);
+    bh.consume(concurrentHashMap);
+
+    Set immutableSet = CollectionUtils.tryMakeImmutableSet(Arrays.asList(DECOY_KEYS));
+    Map immutableMap = CollectionUtils.tryMakeImmutableMap(mapCopySource);
+    for (Object key : DECOY_KEYS) {
+      bh.consume(immutableSet.contains(distinctEqualCopy(key)));
+      bh.consume(immutableMap.get(distinctEqualCopy(key)));
+    }
+    bh.consume(immutableSet);
+    bh.consume(immutableMap);
   }
 
   /**
-   * Counterpart to the rest of this class for the {@code compareTo}-based dispatch used by {@code
-   * TreeMap}/{@code TreeSet}/{@code ConcurrentSkipListMap}, instead of {@code hashCode()}/{@code
-   * equals()}.
+   * Counterpart to {@link #polluteHashDispatch} for the {@code compareTo}-based dispatch used by
+   * {@code TreeMap}/{@code TreeSet}/{@code ConcurrentSkipListMap}, instead of {@code hashCode()}/
+   * {@code equals()}.
    *
    * 
      Unlike the hash-based structures above, a single sorted collection can't hold multiple decoy
    * key classes at once: natural ordering calls {@code key.compareTo(existing)}, which throws
    * {@code ClassCastException} the moment two mutually-incomparable types meet (e.g. a {@code
-   * String} and an {@code Integer}). So each key type gets its own scratch instance here, one type
+   * String} and an {@code Integer}). So each key type gets its own fresh collection here, one type
    * at a time. That's still enough to make the dispatch megamorphic: HotSpot's type profile lives
    * on the bytecode call site inside {@code TreeMap}/{@code TreeSet}/{@code
    * ConcurrentSkipListMap}'s shared implementation, not on any one collection instance, so driving
-   * several receiver types through that site across several scratch instances pollutes it exactly
-   * as effectively as driving them through one shared instance would.
+   * several receiver types through that site across several collection instances pollutes it
+   * exactly as effectively as driving them through one shared instance would.
    *
    * 
      {@code Object}'s decoy is skipped: it isn't {@link Comparable}, so it has no natural
    * ordering to dispatch through in the first place -- the same reason it's exempt from an {@code
    * equals()}-identity copy in {@link #distinctEqualCopy}.
    */
-  public static void polluteCompareToDispatch() {
-    polluteCompareToDispatch(DEFAULT_DECOY_KEYS);
-  }
-
-  public static void polluteCompareToDispatch(Object... decoyKeys) {
-    for (Object key : decoyKeys) {
+  private static void polluteCompareToDispatch(Blackhole bh) {
+    for (Object key : DECOY_KEYS) {
       if (!(key instanceof Comparable)) {
         continue;
       }
+
       TreeSet treeSet = new TreeSet<>();
       treeSet.add(key);
-      treeSet.contains(distinctEqualCopy(key));
+      bh.consume(treeSet.contains(distinctEqualCopy(key)));
+      bh.consume(treeSet);
 
       TreeMap treeMap = new TreeMap<>();
       treeMap.put(key, key);
-      treeMap.get(distinctEqualCopy(key));
+      bh.consume(treeMap.get(distinctEqualCopy(key)));
+      bh.consume(treeMap);
 
       ConcurrentSkipListMap skipListMap = new ConcurrentSkipListMap<>();
       skipListMap.put(key, key);
-      skipListMap.get(distinctEqualCopy(key));
-    }
-  }
-
-  /**
-   * Exercises {@code contains()} with the default decoy keys.
-   *
-   * 
      Use the collection implementation under test; the instance itself may be a scratch object.
-   * HotSpot stores receiver-type profiles at bytecode call sites, so every instance executing that
-   * implementation contributes to the same internal {@code hashCode()} and {@code equals()}
-   * profiles. The collection may be mutable or immutable.
-   */
-  public static void populateTypeProfile(Collection populated) {
-    populateTypeProfile(populated, DEFAULT_DECOY_KEYS);
-  }
-
-  public static void populateTypeProfile(Collection populated, Object... decoyKeys) {
-    for (Object key : decoyKeys) {
-      populated.contains(distinctEqualCopy(key));
-    }
-  }
-
-  /** Exercises {@code add()} and {@code contains()} on a mutable collection. */
-  public static void populateTypeProfileMutable(Collection scratch, Object... decoyKeys) {
-    for (Object key : decoyKeys) {
-      scratch.add(key);
-      scratch.contains(distinctEqualCopy(key));
-    }
-  }
-
-  /**
-   * {@link Map} counterpart to {@link #populateTypeProfile(Collection)}: pass the map instance
-   * under test (or an equivalent scratch instance) to drive its {@code get()} dispatch. Safe
-   * against immutable maps too, since it only calls {@code get()}.
-   */
-  public static void populateTypeProfileMap(Map populated) {
-    populateTypeProfileMap(populated, DEFAULT_DECOY_KEYS);
-  }
-
-  public static void populateTypeProfileMap(Map populated, Object... decoyKeys) {
-    for (Object key : decoyKeys) {
-      populated.get(distinctEqualCopy(key));
-    }
-  }
-
-  /**
-   * Lower-level control, {@link Map} counterpart to {@link #populateTypeProfileMutable}: also
-   * drives {@code put()} dispatch, so {@code scratch} must genuinely support mutation.
-   */
-  public static void populateTypeProfileMutableMap(
-      Map scratch, Object... decoyKeys) {
-    for (Object key : decoyKeys) {
-      scratch.put(key, key);
-      scratch.get(distinctEqualCopy(key));
+      bh.consume(skipListMap.get(distinctEqualCopy(key)));
+      bh.consume(skipListMap);
     }
   }
 
@@ -181,7 +164,16 @@ public static void populateTypeProfileMutableMap(
    * equals()} is identity, so a decoy of that type has no distinct-but-equal instance to make; it's
    * returned as-is, and the identity fast path is then indistinguishable from a genuine {@code
    * equals()} call anyway.
+   *
+   * 
      {@code DONT_INLINE} makes this call boundary an optimization black box: without it, once a
+   * caller like {@link #polluteHashDispatch} is inlined into a tight loop, the JIT can trace a
+   * decoy key's concrete type straight back through this method to the literal it originated from
+   * and devirtualize {@code hashCode()}/{@code equals()} via static type inference alone --
+   * bypassing the shared, runtime type profile this class exists to pollute, no matter how many
+   * distinct instances or types are pushed through it. Keeping this a real, non-inlined call forces
+   * every caller to go through actual dispatch.
    */
+  @CompilerControl(CompilerControl.Mode.DONT_INLINE)
   @SuppressWarnings(
       "deprecation") // boxed-type constructors: only way to force a non-cached instance
   private static Object distinctEqualCopy(Object key) {
diff --git a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
index 52f5485b537..39c6bb169f8 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
@@ -24,61 +24,38 @@
  *       allocation-free (case folded inside hash/matches), value stored unboxed
  * 
  *
- * 
      Takeaways. FlatHashtable is ~2x the (previously recommended) TreeMap at the same zero
- * allocation, and matches HashMap's look-up throughput without HashMap's per-look-up folded
- * String (which drives the multi-threaded GC pressure). The case-insensitive hash is the
- * consistent-for-all-inputs two-way fold ({@link
- * datadog.trace.util.Strings#caseInsensitiveHashCode} — see its note); a cheaper ASCII-only fold
- * would recover a few percent for header-name-only hot paths, deliberately not the default. {@code
- * LOW_LOAD_FACTOR} makes no difference here (the fold, not the probe count, dominates), so the
- * default 0.5 is used.
+ * 
      Takeaways. FlatHashtable is ~1.8x the (previously recommended) TreeMap at the same zero
+ * allocation, but — see the revised takeaway below — trails HashMap's look-up throughput; its case
+ * is the allocation win (no per-look-up folded String, which drives the multi-threaded GC pressure
+ * HashMap pays), not a throughput win. The case-insensitive hash is the consistent-for-all-inputs
+ * two-way fold ({@link datadog.trace.util.Strings#caseInsensitiveHashCode} — see its note); a
+ * cheaper ASCII-only fold would recover a few percent for header-name-only hot paths, deliberately
+ * not the default. {@code LOW_LOAD_FACTOR} makes no difference here (the fold, not the probe count,
+ * dominates), so the default 0.5 is used.
  *
- * 
      Numbers below: MacBook M1, Zulu 21, per-thread lookup index, @Fork(5). 
- * 1 thread
- *
- * Benchmark                                     Mode  Cnt          Score        Error  Units
- * create_flatHashtable                         thrpt   15     2158595.7 ±    73576.7  ops/s
- * create_hashMap                               thrpt   15      944890.9 ±    34398.7  ops/s
- * create_treeMap                               thrpt   15     1285085.3 ±   133648.6  ops/s
- *
- * lookup_flatHashtable                         thrpt   15    75350287.4 ±  4128577.5  ops/s
- * lookup_flatHashtable_lowLoad                 thrpt   15    77127204.7 ±  2546322.0  ops/s
- * lookup_hashMap                               thrpt   15    76615721.1 ±  4615488.0  ops/s
- * lookup_treeMap                               thrpt   15    45777645.5 ±  4551223.1  ops/s
- *  
- * 8 threads (with -prof gc; alloc = gc.alloc.rate.norm)
- *
- * Benchmark                          Mode  Cnt          Score         Error  Units      alloc
- * lookup_flatHashtable              thrpt   15  537007985.7 ±  21864181.3  ops/s     ~0 B/op
- * lookup_flatHashtable_lowLoad      thrpt   15  540434673.5 ±  20451984.4  ops/s     ~0 B/op
- * lookup_hashMap                    thrpt   15  441875038.1 ± 110408182.2  ops/s   24.0 B/op (129 GCs)
- * lookup_treeMap                    thrpt   15  251195415.1 ±  14662568.3  ops/s     ~0 B/op
- * 
- *
- * 
      Rerun on JDK 8 with {@link BenchmarkUtils#polluteHashDispatch()} added to a new
- * {@code @Setup(Level.Trial)} (this file had none before), at this file's actual {@code @Fork(2)}
- * (the numbers above are from an ad hoc higher-fork run; not directly comparable). M ops/s, 8
- * threads:
+ * 
      Java 17 results (MacBook M1, {@code @Fork(5)}, {@code @Threads(8)}) with the front-loaded
+ * {@link BenchmarkUtils#warmUpHashDispatch} pollution design (M ops/s):
  *
  * 
      {@code
- * create_baseline        26    create_flatHashtable   13
- * create_hashMap          9    create_treeMap          7
+ * create_baseline        25.2    create_flatHashtable    15.3
+ * create_hashMap          7.3    create_treeMap           8.4
  *
- * lookup_baseline      2618    lookup_flatHashtable  415
- * lookup_flatHashtable_lowLoad 415  lookup_hashMap    367*
- * lookup_treeMap        209
+ * lookup_baseline       2760.8   lookup_flatHashtable    380.3
+ * lookup_flatHashtable_lowLoad  430.7  lookup_hashMap    488.5
+ * lookup_treeMap         214.7
  * }
      
  *
- * 
      * = error bar over a third of the mean at {@code @Fork(2)} — directional only.
+ * 
      {@code lookup_flatHashtable}/{@code lookup_hashMap}/{@code lookup_treeMap} carry error bars of
+ * ~13-17% of their means at {@code @Fork(5)} (down from 44-66% at {@code @Fork(2)}, which wasn't
+ * decisive) — tight enough that {@code hashMap}'s lead over {@code flatHashtable} (488.5 vs 380.3,
+ * ~28%) is a real, if not perfectly clean-cut, result rather than noise.
  *
- * 
      All four {@code lookup_*} numbers sit 17-23% below the table above (415 vs 537 flatHashtable,
- * 367 vs 442 hashMap, 209 vs 251 treeMap) despite {@code flatHashtable} and {@code treeMap} using
- * neither {@code java.util.HashMap} nor {@code hashCode()}/{@code equals()} dispatch — so this drop
- * isn't attributable to pollution. The likelier explanation: the table above is Zulu 21, this rerun
- * is JDK 8, and JDK 8's C2 backend for Apple Silicon (AArch64) is far less mature than JDK 17+'s —
- * a broad-based slowdown across every entry, pollution-affected or not, is expected from that JDK
- * gap alone on this machine. The relative ranking — {@code flatHashtable} > {@code hashMap}
- * > {@code treeMap} — is unchanged.
+ * 
      Takeaway, revised. {@code HashMap} keyed on {@code toLowerCase()} is faster than {@code
+ * FlatHashtable} for this lookup shape, not merely comparable to it as earlier (noisier) runs
+ * suggested. {@code FlatHashtable} still wins on allocation — it is the zero-allocation option, and
+ * that stays true regardless of the throughput ordering — but the throughput case for it over
+ * {@code HashMap} on case-insensitive lookups does not hold up under this rerun. {@code TreeMap}
+ * remains the slowest of the three at every fork count measured.
  */
 @Fork(2)
 @Warmup(iterations = 2)
@@ -128,13 +105,12 @@ static  T init(Supplier supplier) {
   // masking exactly the differences this benchmark compares.
   int lookupIndex = 0;
 
-  // Re-pollute every invocation: a one-shot Level.Trial call gets drowned out by this
-  // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
-  // sites, letting them re-specialize to a dominant receiver (see
-  // BenchmarkUtils#polluteHashDispatch).
-  @Setup(Level.Invocation)
-  public void pollute() {
-    BenchmarkUtils.polluteHashDispatch();
+  // Front-load pollution once per trial, entirely before JMH's warmup starts: JMH
+  // injects the Blackhole straight into this setup method, so no per-benchmark
+  // scratch state is needed.
+  @Setup(Level.Trial)
+  public void warmUpPollution(Blackhole bh) {
+    BenchmarkUtils.warmUpHashDispatch(bh);
   }
 
   String nextLookupKey() {
diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
index d16ee3c4eca..28998f7ef48 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
@@ -59,15 +59,15 @@
  * HashtableD1Benchmark.iterate_hashtable  thrpt    6    22.208 ±   0.956  ops/us
  * 
  *
- * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} added to {@code D1State.setUp()} (same
+ * 
      Rerun with {@link BenchmarkUtils#warmUpHashDispatch} added to {@code D1State.setUp()} (same
  * machine/JVM/config): every number moved down somewhat (add_hashMap 188→101, update_hashtable
  * 1810→1465, iterate_hashtable 22→17 ops/us), including {@code *_hashtable}. That's expected to be
  * a no-op for {@code *_hashtable}: {@link Hashtable.D1.Entry#hash} and {@link
  * Hashtable.D1.Entry#matches} are call sites private to {@code Hashtable.java}, structurally
  * distinct from {@code java.util.HashMap}/{@code HashSet}'s internal {@code hashCode()}/{@code
- * equals()} call sites — JIT type profiles are keyed per call site, so {@code
- * polluteHashDispatch()} cannot reach them regardless of key-type overlap. Since the JDK and
- * machine were held constant across this rerun (unlike the JDK 8-vs-17 comparisons in {@link
+ * equals()} call sites — JIT type profiles are keyed per call site, so {@code warmUpHashDispatch}
+ * cannot reach them regardless of key-type overlap. Since the JDK and machine were held constant
+ * across this rerun (unlike the JDK 8-vs-17 comparisons in {@link
  * datadog.trace.util.CaseInsensitiveMapBenchmark} and {@link
  * datadog.trace.api.TagMapAccessBenchmark}), the drop here is same-session run-to-run noise
  * (thermal/power, not controlled for) rather than either a pollution effect or a JDK effect. The
@@ -143,10 +143,16 @@ public static class D1State {
     int cursor;
     final BhD1Consumer consumer = new BhD1Consumer();
 
+    // Front-load pollution once per trial, entirely before JMH's warmup starts: JMH
+    // injects the Blackhole straight into this setup method, so no per-benchmark
+    // scratch state is needed.
+    @Setup(Level.Trial)
+    public void warmUpPollution(Blackhole bh) {
+      BenchmarkUtils.warmUpHashDispatch(bh);
+    }
+
     @Setup(Level.Iteration)
     public void setUp() {
-      BenchmarkUtils.polluteHashDispatch();
-
       table = new Hashtable.D1<>(CAPACITY);
       hashMap = new HashMap<>(CAPACITY);
       keys = SOURCE_KEYS;
diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
index 1a181bee77c..ae8510890ba 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
@@ -62,7 +62,7 @@
  * HashtableD2Benchmark.iterate_hashtable  thrpt    6    16.968 ±   0.371  ops/us
  * 
  *
- * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} added to {@code D2State.setUp()} (same
+ * 
      Rerun with {@link BenchmarkUtils#warmUpHashDispatch} added to {@code D2State.setUp()} (same
  * machine/JVM/config): results were noisy and inconsistent with a clean pollution story —
  * add_hashMap actually rose (77→103), while add_hashtable fell sharply (217→118, error bars wider
  * than the mean both times); update_hashtable fell (1446→1225) and both iterate numbers fell
@@ -179,10 +179,16 @@ public static class D2State {
     int cursor;
     final BhD2Consumer consumer = new BhD2Consumer();
 
+    // Front-load pollution once per trial, entirely before JMH's warmup starts: JMH
+    // injects the Blackhole straight into this setup method, so no per-benchmark
+    // scratch state is needed.
+    @Setup(Level.Trial)
+    public void warmUpPollution(Blackhole bh) {
+      BenchmarkUtils.warmUpHashDispatch(bh);
+    }
+
     @Setup(Level.Iteration)
     public void setUp() {
-      BenchmarkUtils.polluteHashDispatch();
-
       table = new Hashtable.D2<>(CAPACITY);
       hashMap = new HashMap<>(CAPACITY);
       k1s = SOURCE_K1;
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
index dc2f7ad035a..c64826f7176 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
@@ -44,7 +44,7 @@
  * {@code *_sameKey} variants reuse the original interned key instances to show the identity fast
  * path — which is the common tracer case, since map keys are typically interned tag-name constants.
  *
- * 
      {@link BenchmarkUtils#polluteHashDispatch()} prepares shared {@code hashCode()} and {@code
+ * 
      {@link BenchmarkUtils#warmUpHashDispatch} prepares shared {@code hashCode()} and {@code
  * equals()} call sites before measurement. This avoids giving hash-based structures an
  * unrealistically monomorphic type profile.
  *
@@ -53,29 +53,29 @@
  * set dominates. HotSpot can usually devirtualize and inline the monomorphic case, sometimes a
  * small polymorphic one; megamorphic sites generally retain virtual dispatch.
  *
- * 
      Results on an Apple M1 with Java 8u382, {@code @Fork(5)}, and {@code @Threads(8)} (M ops/s):
+ * 
      Java 17 results on an Apple M1 with the front-loaded {@link BenchmarkUtils#warmUpHashDispatch}
+ * pollution design, {@code @Fork(5)}, {@code @Threads(8)} (M ops/s):
  *
  * 
      {@code
  * Structure                get sameKey iterate iterate_forEach
- * hashMap                 1202    1438     120        -
- * linkedHashMap           1097       -     127        -
- * treeMap                  487       -     121        -
- * tagMap                  1005    1235     109       121
- * tracerImmutableMap      1052    1249     123        -   (MapN)
- * stringIndex             1366    1724       -        -
- * stringIndex_embedded    1479    1846       -        -
+ * hashMap                1240.9  1827.4  109.95        -
+ * linkedHashMap          1259.5     -    139.86        -
+ * treeMap                 665.0     -    139.49        -
+ * tagMap                 1200.3  1420.9  100.66     146.69
+ * tracerImmutableMap     1067.3  1395.9  137.93        -   (MapN)
+ * stringIndex             1452.4  1864.8       -        -
+ * stringIndex_embedded    1605.2  2063.5       -        -
  * }
      
  *
  * 
      In this run:
  *
  * 
        
- *   
      • The embedded StringIndex has the fastest {@code get}; the instance wrapper is second.
- *   
      • Both StringIndex variants outperform the map-based alternatives for distinct and identical
- *       key instances.
- *   
      • {@code TreeMap.get} varies widely across forks (roughly 230-610 M ops/s), so its mean is
- *       less stable than the other results.
- *   
      • {@code TagMap.forEach} is about 10% faster than its iterator (121 vs. 109 M ops/s). Its
- *       advantage widens as TagMap's entry model grows.
+ *   
      • The embedded StringIndex has the fastest {@code get}; the instance wrapper is second — both
+ *       StringIndex variants outperform the map-based alternatives for distinct and identical key
+ *       instances.
+ *   
      • {@code stringIndex} is also slightly better than {@code hashMap} when used as an immutable
+ *       map (1452.4 vs 1240.9 on {@code get}).
+ *   
      • {@code TagMap.forEach} is about 46% faster than its iterator (146.69 vs. 100.66 M ops/s).
  * 
      
  */
 // @Fork(5): get_tracerImmutableMap* (MapN reached via interface dispatch) is JIT-bimodal at fewer
@@ -139,8 +139,9 @@ static void fill(Map map) {
 
   @Setup(Level.Trial)
   public void setUp() {
-    BenchmarkUtils.polluteHashDispatch();
-
+    // Superseded by Cursor#warmUpPollution's heavier front-load below -- this single call isn't
+    // enough on its own to drive HotSpot's tiered compiler through both C1 and C2 on the shared
+    // hash-dispatch call sites (see BenchmarkUtils#warmUpHashDispatch).
     hashMap = new HashMap<>();
     fill(hashMap);
     linkedHashMap = new LinkedHashMap<>();
@@ -161,13 +162,12 @@ public void setUp() {
   public static class Cursor {
     int index = 0;
 
-    // Re-pollute every invocation: a one-shot Level.Trial call gets drowned out by this
-    // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
-    // sites, letting them re-specialize to a dominant receiver (see
-    // BenchmarkUtils#polluteHashDispatch).
-    @Setup(Level.Invocation)
-    public void pollute() {
-      BenchmarkUtils.polluteHashDispatch();
+    // Front-load pollution once per trial, entirely before JMH's warmup starts: JMH
+    // injects the Blackhole straight into this setup method, so no per-benchmark
+    // scratch state is needed.
+    @Setup(Level.Trial)
+    public void warmUpPollution(Blackhole bh) {
+      BenchmarkUtils.warmUpHashDispatch(bh);
     }
 
     String nextKey() {
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java
index 9b322e11af9..6e9bccf5398 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java
@@ -13,6 +13,7 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
 
 /**
  * Membership over a small, fixed, read-only string set shared across threads — split into hit and
@@ -54,43 +55,33 @@
  *   
    • {@code miss} uses non-interned strings that are not in the set.
  * 
  *
- * 
      Results on an Apple M1 with Java 8u382, {@link BenchmarkUtils#polluteHashDispatch()} enabled,
- * {@code @Fork(5)}, and {@code @Threads(8)} (M ops/s):
+ * 
      Java 17 results on an Apple M1 with the front-loaded {@link BenchmarkUtils#warmUpHashDispatch}
+ * pollution design, {@code @Fork(5)}, {@code @Threads(8)} (M ops/s):
  *
  * 
      {@code
  * Structure                    hit   hitFresh    miss
- * stringIndex_embedded (static) 2098      1563    2030
- * hashSet                       1723      1276    1823
- * stringIndex (inst)            1883      1184 *  1700 *
- * tracerImmutableSet            1632      1232    1625    (SetN)
- * array                          854         -     495
- * sortedArray                    713         -     613
- * treeSet                        646         -     544
+ * stringIndex_embedded        2231.9   1663.2   2207.7
+ * hashSet                     2172.8   1359.2   2252.9
+ * tracerImmutableSet          2045.4   1394.9   1711.8
+ * stringIndex (inst)          2037.6   1505.7   2060.7
+ * array                        967.4        -    611.0
+ * sortedArray                  691.4        -    598.6
+ * treeSet                      657.2        -    607.2
  * }
      
  *
  * 
      In this run:
  *
  * 
        
- *   
      • The embedded {@code StringIndex} is fastest for all three lookup variants.
- *   
      • The {@code StringIndex} wrapper beats {@code HashSet} for interned hits. Its fresh-hit and
- *       miss results are bimodal and have lower means than {@code HashSet}; prefer the embedded
- *       form when these paths matter.
- *   
      • {@code SetN} is slower than the embedded form but about 27% smaller. StringIndex trades
- *       that space for speed and support for slot-aligned payload arrays.
- *   
      • Fresh hits are slower than misses for each hash-based structure: a matching distinct string
- *       reaches {@code equals()}, while a miss can stop on a hash mismatch.
+ *   
      • {@code stringIndex} is slightly better than {@code hashSet} on {@code hitFresh}, but not
+ *       uniformly ahead across all three lookup variants the way earlier runs suggested; the two
+ *       trade the lead depending on the variant.
+ *   
      • The embedded {@code StringIndex} form remains at or near the front for all three lookup
+ *       variants, and is now about as fast as the instance wrapper rather than clearly ahead of it.
+ *   
      • {@code tracerImmutableSet} ({@code SetN}) is competitive with the hash-based structures on
+ *       {@code hit}/{@code hitFresh} but falls behind on {@code miss}.
  * 
      
- *
- * 
      Caveat — the instance {@code stringIndex} miss is bimodal across forks (confirmed at
- * {@code @Fork(10)}: 6 forks fast, 4 slow, nothing between). ~60% of forks compile to a fast mode
- * (~2000, ≈ {@code stringIndex_embedded_miss} — the wrapper indirection is then free) and ~40% to a
- * slow mode (~1070, ~half); each fork locks one at warmup. So the {@code 1548 ±27%} above is a
- * mode-mix, not noise. Cause: C2 hoists the instance field-loads ({@code this.hashes}/{@code
- * names}) out of the miss-path probe loop only in the fast mode; the static {@code
- * EmbeddingSupport} path const-folds those refs and is never bimodal ({@code
- * stringIndex_embedded_miss} ±0.3%). Prefer {@code EmbeddingSupport} where miss latency matters.
  */
-@Fork(5) // 5 forks settle the bimodal stringIndex_miss / interface-dispatch arms (see header)
+@Fork(5) // extra forks needed historically to settle bimodal JIT behavior on some arms
 @Warmup(iterations = 2)
 @Measurement(iterations = 3)
 @Threads(8)
@@ -146,8 +137,9 @@ static String[] newMisses() {
 
   @Setup(Level.Trial)
   public void setUp() {
-    BenchmarkUtils.polluteHashDispatch();
-
+    // Superseded by Cursor#warmUpPollution's heavier front-load below -- this single call isn't
+    // enough on its own to drive HotSpot's tiered compiler through both C1 and C2 on the shared
+    // hash-dispatch call sites (see BenchmarkUtils#warmUpHashDispatch).
     array = STRINGS;
     sortedArray = Arrays.copyOf(STRINGS, STRINGS.length);
     Arrays.sort(sortedArray);
@@ -164,13 +156,12 @@ public static class Cursor {
     int hitFreshIndex = 0;
     int missIndex = 0;
 
-    // Re-pollute every invocation: a one-shot Level.Trial call gets drowned out by this
-    // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
-    // sites, letting them re-specialize to a dominant receiver (see
-    // BenchmarkUtils#polluteHashDispatch).
-    @Setup(Level.Invocation)
-    public void pollute() {
-      BenchmarkUtils.polluteHashDispatch();
+    // Front-load pollution once per trial, entirely before JMH's warmup starts: JMH
+    // injects the Blackhole straight into this setup method, so no per-benchmark
+    // scratch state is needed.
+    @Setup(Level.Trial)
+    public void warmUpPollution(Blackhole bh) {
+      BenchmarkUtils.warmUpHashDispatch(bh);
     }
 
     String nextHit() {
diff --git a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java
index 36f824c5aad..51ade692ce6 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java
@@ -51,46 +51,39 @@
  * the tax is the delta to the {@code synchronizedHashMap} equivalents. Comparing across JVM
  * versions at stock flags shows the biased-locking effect. (Results pending a fresh multi-JVM run.)
  *
- * 
      JDK 8 results (Apple M1, {@code @Fork(2)}, {@code @Threads(8)}, with {@link
- * BenchmarkUtils#polluteHashDispatch()} in effect; M ops/s):
+ * 
      Java 17 results (Apple M1, {@code @Fork(2)}, {@code @Threads(8)}) with the front-loaded {@link
+ * BenchmarkUtils#warmUpHashDispatch} pollution design (M ops/s):
  *
  * 
      {@code
- * create_hashMap                 79   create_hashMap_sized     38*
- * create_synchronizedHashMap    8.6   create_treeMap            5*
- * create_linkedHashMap            8*  create_tagMap            10
- * create_tagMap_via_ledger        9   create_flatHashtable    190
+ * create_flatHashtable          183.4   create_hashMap                 104.7
+ * create_hashMap_sized          110.3   create_linkedHashMap            57.4
+ * create_synchronizedHashMap     52.0   create_tagMap                  103.6
+ * create_tagMap_via_ledger       76.2   create_treeMap                  34.6
  *
- * clone_hashMap                  68*  clone_synchronizedHashMap 58
- * clone_treeMap                 100   clone_linkedHashMap       94
- * clone_tagMap                  249
+ * clone_tagMap                  301.8   clone_treeMap                  101.5
+ * clone_hashMap                  61.2   clone_synchronizedHashMap       54.9
+ * clone_linkedHashMap            50.8
  *
- * get_hashMap                   164   get_synchronizedHashMap   67
- * get_flatHashtable              196
+ * get_flatHashtable             1583.7  get_hashMap                    1352.1
+ * get_synchronizedHashMap        843.2
  *
- * iterate_hashMap                119  iterate_synchronizedHashMap 81
- * iterate_flatHashtable           14*
+ * iterate_flatHashtable          182.6  iterate_hashMap                 106.3
+ * iterate_synchronizedHashMap     93.2
  * }
      
  *
- * 
      * = error bar as wide as (or wider than) the mean at {@code @Fork(2)} — treat these as
- * directional, not decisive; a {@code @Fork(5)} rerun would tighten them (see {@code
- * ThreadSafeMapBenchmark}'s Javadoc for the same caveat pattern). The construction benchmarks are
- * consistently the noisy ones; the read/clone/iterate benchmarks are comparatively tight.
- *
  * 
      Key findings:
  *
  * 
        
- *   
      • {@code flatHashtable} dominates both {@code create} (190M) and {@code get} (196M) — the
+ *   
      • {@code flatHashtable} dominates {@code create}, {@code get}, and {@code iterate} — the
  *       unboxed, self-contained entry and comparison-free insert pay off, consistent with every
  *       other FlatHashtable comparison in this module.
- *   
      • {@code tagMap} clone (249M) is ~3.7x {@code hashMap} clone (68M) — the same story {@link
- *       datadog.trace.api.TagMapAccessBenchmark} reports from an earlier (unpolluted, Java 17) run
- *       at ~4.6x; the ratio survives pollution and a different JDK, even though the absolute
- *       numbers aren't directly comparable across those two runs.
- *   
      • The uncontended synchronization tax is large here even though this run is on JDK 8, where
- *       biased locking is enabled by default: {@code get_hashMap} (164M) → {@code
- *       get_synchronizedHashMap} (67M) is a ~59% hit, and {@code iterate} (119M → 81M) is ~32%.
- *       That's a bigger tax than the "biased locking should make uncontended locking nearly free"
- *       story predicts — not root-caused here, left as an open question rather than papered over.
+ *   
      • {@code tagMap} clone (301.8M) is ~4.9x {@code hashMap} clone (61.2M) — the same story
+ *       {@link datadog.trace.api.TagMapAccessBenchmark} reports, by design ({@code TagMap} clone is
+ *       a purpose-built fast path).
+ *   
      • The uncontended synchronization tax is visible even with no contention, consistent with
+ *       Java 15+'s biased locking being disabled by default (JEP 374): {@code get_hashMap}
+ *       (1352.1M) → {@code get_synchronizedHashMap} (843.2M) is a ~38% hit, and {@code iterate}
+ *       (106.3M → 93.2M) is ~12%.
  * 
      
  */
 @Fork(2)
@@ -231,17 +224,13 @@ static IntEntry[] newFilledFlat() {
   IntEntry[] flatTable;
   int index = 0;
 
-  // Re-pollute every invocation: a one-shot Level.Trial call gets drowned out by this
-  // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
-  // sites, letting them re-specialize to a dominant receiver (see
-  // BenchmarkUtils#polluteHashDispatch).
-  @Setup(Level.Invocation)
-  public void pollute() {
-    BenchmarkUtils.polluteHashDispatch();
-  }
-
+  // Front-load pollution once per trial, entirely before JMH's warmup starts: JMH
+  // injects the Blackhole straight into this setup method, so no per-benchmark
+  // scratch state is needed.
   @Setup(Level.Trial)
-  public void setUp() {
+  public void setUp(Blackhole bh) {
+    BenchmarkUtils.warmUpHashDispatch(bh);
+
     hashMap = new HashMap<>();
     fill(hashMap);
     synchronizedHashMap = Collections.synchronizedMap(new HashMap<>(hashMap));
diff --git a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java
index 04cb4c43034..c1ba965702d 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedSetBenchmark.java
@@ -30,54 +30,31 @@
  * CAS on Java 15+ (biased locking disabled by default, JEP 374). The unsynchronized {@code hashSet}
  * {@code contains}/{@code iterate} methods are the in-harness baseline; the tax is the delta.
  *
- * 
      Java 17 results (Apple M1, {@code @Fork(2)}, {@code @Threads(8)}; M ops/s = millions):
+ * 
      Java 17 results (Apple M1, {@code @Fork(2)}, {@code @Threads(8)}) with the front-loaded {@link
+ * BenchmarkUtils#warmUpHashDispatch} pollution design (M ops/s = millions):
  *
  * 
      {@code
- * contains_hashSet            1291
- * contains_synchronizedSet     808    (~37% slower — the uncontended sync tax)
- * iterate_hashSet              91
- * iterate_synchronizedSet      90    (one monitor acquire amortized over the walk)
+ * contains_hashSet            1376.4
+ * contains_synchronizedSet     814.5    (~41% slower — the uncontended sync tax)
+ * iterate_hashSet               94.6
+ * iterate_synchronizedSet       91.3    (one monitor acquire amortized over the walk)
  *
- * create_hashSet         81    clone_hashSet          48
- * create_hashSet_sized   78    clone_synchronizedSet  47
- * create_linkedHashSet   61    clone_linkedHashSet    59
- * create_synchronizedSet 41    clone_treeSet          83
- * create_treeSet         36
+ * create_hashSet          94.4   clone_hashSet          48.4
+ * create_hashSet_sized    95.9   clone_synchronizedSet  44.0
+ * create_linkedHashSet    41.3   clone_linkedHashSet    42.7
+ * create_synchronizedSet  41.7   clone_treeSet          86.1
+ * create_treeSet          32.7
  * }
      
  *
- * 
      JDK 8 results (Apple M1, {@code @Fork(2)}, {@code @Threads(8)}, with {@link
- * BenchmarkUtils#polluteHashDispatch()} in effect; M ops/s):
- *
- * 
      {@code
- * contains_hashSet            1421
- * contains_synchronizedSet     746    (~48% slower — the uncontended sync tax)
- * iterate_hashSet              134
- * iterate_synchronizedSet      129    (one monitor acquire amortized over the walk)
- *
- * create_hashSet         67    clone_hashSet          56
- * create_hashSet_sized   83*   clone_synchronizedSet  48*
- * create_linkedHashSet   63    clone_linkedHashSet    56
- * create_synchronizedSet 71*   clone_treeSet          77
- * create_treeSet         38
- * }
      
- *
- * 
      * = error bar over half the mean at {@code @Fork(2)} — directional only.
- *
  * 
      Key findings:
  *
  * 
        
- *   
      • Uncontended synchronization tax holds up under pollution and on a different JDK:
- *       {@code contains} is ~48% slower synchronized (1421 → 746M ops/s on JDK 8, vs. ~37% on Java
- *       17) — same story, somewhat larger tax. {@code iterate}'s tax stays small either way (~4%
- *       here): one monitor acquire amortized over the walk.
- *   
      • {@code contains_hashSet} and {@code iterate_hashSet} land in the same range as the original
- *       Java 17 run (1291 vs 1421M, 91 vs 134M) rather than collapsing, unlike {@link
- *       ImmutableSetBenchmark}'s {@code hitFresh} case -- but that run is also on a different JDK,
- *       so it isn't a clean pollution-only comparison (same caveat as {@link
- *       HashtableD1Benchmark}'s Javadoc); read it as weak, not decisive, evidence that pollution
- *       didn't change the qualitative story. Construction numbers remain the noisiest (several
- *       {@code @Fork(2)} error bars exceed half the mean); {@code TreeSet} stays the slowest to
- *       build across both runs.
+ *   
      • Uncontended synchronization tax holds up under pollution: {@code contains} is ~41%
+ *       slower synchronized (1376.4 → 814.5M ops/s), consistent with Java 15+'s biased locking
+ *       being disabled by default (JEP 374). {@code iterate}'s tax stays small (~3.5%): one monitor
+ *       acquire amortized over the walk.
+ *   
      • {@code TreeSet} stays the slowest structure to build but is, notably, the fastest to clone
+ *       (86.1M) — worth a closer look if that gap turns out to matter elsewhere.
  * 
      
  */
 @Fork(2)
@@ -115,17 +92,13 @@ static void fill(Set set) {
   LinkedHashSet linkedHashSet;
   int index = 0;
 
-  // Re-pollute every invocation: a one-shot Level.Trial call gets drowned out by this
-  // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
-  // sites, letting them re-specialize to a dominant receiver (see
-  // BenchmarkUtils#polluteHashDispatch).
-  @Setup(Level.Invocation)
-  public void pollute() {
-    BenchmarkUtils.polluteHashDispatch();
-  }
-
+  // Front-load pollution once per trial, entirely before JMH's warmup starts: JMH
+  // injects the Blackhole straight into this setup method, so no per-benchmark
+  // scratch state is needed.
   @Setup(Level.Trial)
-  public void setUp() {
+  public void setUp(Blackhole bh) {
+    BenchmarkUtils.warmUpHashDispatch(bh);
+
     hashSet = new HashSet<>(Arrays.asList(ELEMENTS));
     synchronizedSet = Collections.synchronizedSet(new HashSet<>(hashSet));
     treeSet = new TreeSet<>(Arrays.asList(ELEMENTS));
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java
index 15c131dbd62..0dbbdb3f5b9 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java
@@ -18,6 +18,7 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
 
 /**
  * Measures lookup followed by an atomic counter increment in a shared, pre-populated table. Models
@@ -54,8 +55,8 @@
  *       penalty.
  * 
  *
- * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} wired into {@code
- * SharedState.setUp()}, same JDK 17 as the table above -- a clean pollution-only delta:
+ * 
      Rerun with {@link BenchmarkUtils#warmUpHashDispatch} wired into {@code SharedState.setUp()},
+ * same JDK 17 as the table above -- a clean pollution-only delta:
  *
  * 
      {@code
  * Benchmark                          Score   Units
@@ -114,9 +115,16 @@ public static class SharedState {
     ConcurrentHashMap atomicLongMap;
     ConcurrentHashMap longAdderMap;
 
+    // Front-load pollution once per trial, entirely before JMH's warmup starts: JMH
+    // injects the Blackhole straight into this setup method, so no per-benchmark
+    // scratch state is needed.
+    @Setup(Level.Trial)
+    public void warmUpPollution(Blackhole bh) {
+      BenchmarkUtils.warmUpHashDispatch(bh);
+    }
+
     @Setup(Level.Iteration)
     public void setUp() {
-      BenchmarkUtils.polluteHashDispatch();
       table = ConcurrentHashtable.D1.createBounded(CounterEntry.class, CAPACITY);
       atomicLongMap = new ConcurrentHashMap<>(CAPACITY);
       longAdderMap = new ConcurrentHashMap<>(CAPACITY);
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
index 6c63ea17249..600cc5152df 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
@@ -19,6 +19,7 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
 
 /**
  * Measures steady-state single-key lookups in a shared, pre-populated table.
@@ -36,57 +37,35 @@
  * its {@code _sameKey} vs default variants). See {@link ThreadSafeMapD2Benchmark} for composite
  * keys.
  *
- * 
      Java 17 results ({@code @Fork(2)}, {@code @Threads(8)}, 64 pre-populated keys):
+ * 
      Java 17 results with the front-loaded {@link BenchmarkUtils#warmUpHashDispatch} pollution
+ * design ({@code @Fork(2)}, {@code @Threads(8)}, 64 pre-populated keys; ops/us):
  *
  * 
      {@code
  * Benchmark                             Score   Units
- * get_concurrentHashtable               1583   ops/us
- * get_concurrentHashMap                 1145   ops/us
- * get_concurrentSkipListMap              170   ops/us
- * get_synchronizedHashMap                 33   ops/us
+ * get_concurrentHashtable               1405.8 ops/us
+ * get_concurrentHashMap                 1167.3 ops/us
+ * get_concurrentSkipListMap              193.3 ops/us
+ * get_synchronizedHashMap                  9.2 ops/us
  *
- * getOrCreate_concurrentHashtable       1450   ops/us
- * getOrCreate_concurrentHashMap         1125   ops/us
- * getOrCreate_synchronizedHashMap         31   ops/us
+ * getOrCreate_concurrentHashtable       1495.2 ops/us
+ * getOrCreate_concurrentHashMap         1186.6 ops/us
+ * getOrCreate_synchronizedHashMap          8.9 ops/us
  * }
      
  *
  * 
      Key findings:
  *
  * 
        
- *   
      • {@code ConcurrentHashtable} is ~38% faster than {@code ConcurrentHashMap} on {@code get}
- *       (1583 vs 1145 ops/us); avoids the hash-to-segment translation CHM pays even on its fast
+ *   
      • {@code ConcurrentHashtable} is ~20% faster than {@code ConcurrentHashMap} on {@code get}
+ *       (1405.8 vs 1167.3 ops/us); avoids the hash-to-segment translation CHM pays even on its fast
  *       path.
- *   
      • {@code ConcurrentSkipListMap} is ~9× slower than {@code ConcurrentHashMap} — tree traversal
+ *   
      • {@code ConcurrentSkipListMap} is ~6× slower than {@code ConcurrentHashMap} — tree traversal
  *       cost is high even under lock-free CAS.
- *   
      • Synchronized {@code HashMap} is ~47× slower than {@code ConcurrentHashtable}; the global
- *       lock serializes all 8 threads.
+ *   
      • Synchronized {@code HashMap} is over 150× slower than {@code ConcurrentHashtable}; under
+ *       pollution its megamorphic {@code hashCode()}/{@code equals()} dispatch dominates its cost
+ *       far more than the lock contention this benchmark was designed to isolate.
  *   
      • {@code getOrCreate} is near-identical to {@code get} because all keys are pre-populated —
  *       the lock branch is never taken during measurement.
  * 
      
- *
- * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} wired into {@code
- * SharedState.setUp()}, same JDK 17 as the table above -- a clean pollution-only delta:
- *
- * 
      {@code
- * Benchmark                             Score   Units
- * get_concurrentHashtable               1478   ops/us
- * get_concurrentHashMap                 1188   ops/us
- * get_concurrentSkipListMap              207   ops/us
- * get_synchronizedHashMap                  9   ops/us
- *
- * getOrCreate_concurrentHashtable       1549   ops/us
- * getOrCreate_concurrentHashMap         1188   ops/us
- * getOrCreate_synchronizedHashMap          9   ops/us
- * }
      
- *
- * 
      Synchronized {@code HashMap} collapses by ~73% (33/31 to 9 ops/us on {@code get}/{@code
- * getOrCreate}) -- pollution turns its megamorphic {@code hashCode()}/{@code equals()} dispatch
- * into most of its cost, far more than the lock contention this benchmark was designed to isolate.
- * {@code ConcurrentHashtable} and {@code ConcurrentHashMap} both hold roughly steady (within ~7%),
- * so the {@code ConcurrentHashtable} lead over {@code ConcurrentHashMap} narrows only slightly
- * (~24-30%, down from ~38%) -- this table's own low error bars (all under 2% of their means) make
- * that narrowing a real, if modest, effect rather than noise. {@code ConcurrentSkipListMap} rises
- * slightly (~22%) but with an error bar spanning ~21% of its mean -- directional, not decisive.
  */
 @Fork(2)
 @Warmup(iterations = 2)
@@ -128,7 +107,6 @@ public static class SharedState {
 
     @Setup(Level.Iteration)
     public void setUp() {
-      BenchmarkUtils.polluteHashDispatch();
       table = ConcurrentHashtable.D1.createBounded(D1Entry.class, CAPACITY);
       concurrentHashMap = new ConcurrentHashMap<>(CAPACITY);
       skipListMap = new ConcurrentSkipListMap<>();
@@ -147,13 +125,12 @@ public void setUp() {
   public static class ThreadState {
     int cursor;
 
-    // Re-pollute every invocation: a one-shot Level.Iteration call gets drowned out by this
-    // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
-    // sites, letting them re-specialize to a dominant receiver (see
-    // BenchmarkUtils#polluteHashDispatch).
-    @Setup(Level.Invocation)
-    public void pollute() {
-      BenchmarkUtils.polluteHashDispatch();
+    // Front-load pollution once per trial, entirely before JMH's warmup starts: JMH
+    // injects the Blackhole straight into this setup method, so no per-benchmark
+    // scratch state is needed.
+    @Setup(Level.Trial)
+    public void warmUpPollution(Blackhole bh) {
+      BenchmarkUtils.warmUpHashDispatch(bh);
     }
 
     int next() {
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
index aa3f18d3222..15448be5343 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
@@ -20,6 +20,7 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
 
 /**
  * Measures steady-state composite-key lookups in a shared, pre-populated table.
@@ -39,68 +40,40 @@
  * 
      Lookups reuse the key-part instances installed during setup, taking the identity fast path for
  * their object comparisons. See {@link ThreadSafeMapD1Benchmark} for single-key lookups.
  *
- * 
      Java 17 results ({@code @Fork(2)}, {@code @Threads(8)}, 64 pre-populated keys):
+ * 
      Java 17 results with the front-loaded {@link BenchmarkUtils#warmUpHashDispatch} pollution
+ * design ({@code @Fork(2)}, {@code @Threads(8)}, 64 pre-populated keys; ops/us):
  *
  * 
      {@code
  * Benchmark                              Score   Units
- * get_concurrentHashtable                1452   ops/us
- * get_support                            1450   ops/us
- * get_concurrentHashMap                   777   ops/us
- * get_concurrentSkipListMap               146   ops/us
- * get_synchronizedHashMap                  27   ops/us
+ * get_support                           1573.5   ops/us
+ * get_concurrentHashtable               1412.7   ops/us
+ * get_concurrentHashMap                 1055.3   ops/us
+ * get_concurrentSkipListMap              172.5   ops/us
+ * get_synchronizedHashMap                  8.8   ops/us
  *
- * getOrCreate_support                    1379   ops/us
- * getOrCreate_concurrentHashtable        1119   ops/us
- * getOrCreate_concurrentHashMap           769   ops/us
- * getOrCreate_concurrentSkipListMap       151   ops/us
- * getOrCreate_synchronizedHashMap          28   ops/us
+ * getOrCreate_support                   1492.8   ops/us
+ * getOrCreate_concurrentHashtable       1416.7   ops/us
+ * getOrCreate_concurrentHashMap         1083.2   ops/us
+ * getOrCreate_concurrentSkipListMap      169.9   ops/us
+ * getOrCreate_synchronizedHashMap          8.6   ops/us
  * }
      
  *
  * 
      Key findings:
  *
  * 
        
- *   
      • {@code ConcurrentHashtable} and {@code Support} are neck-and-neck on {@code get} (1452 vs
- *       1450 ops/us); both avoid the {@link Key2} wrapper allocation that {@code ConcurrentHashMap}
- *       requires on every lookup.
- *   
      • {@code ConcurrentHashMap} is ~2× slower than {@code ConcurrentHashtable} on {@code get}
- *       (777 vs 1452 ops/us) — the {@link Key2} allocation plus two-level hash lookup adds up.
- *   
      • {@code Support} shows slightly higher {@code getOrCreate} throughput than {@code D2} (1379
- *       vs 1119 ops/us) because its primitive {@code int} K2 field avoids boxing inside the entry
- *       match on the write-path re-check.
- *   
      • {@code ConcurrentSkipListMap} is ~5× slower than {@code ConcurrentHashMap} due to tree
- *       traversal; the two-traversal {@code getOrCreate} pattern adds further overhead on misses.
- *   
      • Synchronized {@code HashMap} is ~50× slower than {@code ConcurrentHashtable}.
+ *   
      • {@code Support} and {@code ConcurrentHashtable} are the two fastest and stay close to each
+ *       other on both {@code get} and {@code getOrCreate}; both avoid the {@link Key2} wrapper
+ *       allocation that {@code ConcurrentHashMap} requires on every lookup.
+ *   
      • {@code ConcurrentHashMap} is ~30-35% slower than {@code ConcurrentHashtable} — the {@link
+ *       Key2} allocation plus two-level hash lookup adds up.
+ *   
      • {@code Support} shows slightly higher throughput than {@code D2} because its primitive
+ *       {@code int} K2 field avoids boxing inside the entry match on the write-path re-check.
+ *   
      • {@code ConcurrentSkipListMap} is ~6× slower than {@code ConcurrentHashMap} due to tree
+ *       traversal.
+ *   
      • Synchronized {@code HashMap} is over 150× slower than the fastest options; under pollution
+ *       its megamorphic {@code hashCode()}/{@code equals()} dispatch dominates its cost far more
+ *       than lock contention, the same magnitude seen in {@link ThreadSafeMapD1Benchmark}.
  * 
      
- *
- * 
      Rerun with {@link BenchmarkUtils#polluteHashDispatch()} wired into {@code
- * SharedState.setUp()}, same JDK 17 as the table above -- a clean pollution-only delta:
- *
- * 
      {@code
- * Benchmark                              Score   Units
- * get_concurrentHashtable                1505   ops/us
- * get_support                            1313   ops/us
- * get_concurrentHashMap                  1072   ops/us
- * get_concurrentSkipListMap               172   ops/us
- * get_synchronizedHashMap                   9   ops/us
- *
- * getOrCreate_support                    1516   ops/us
- * getOrCreate_concurrentHashtable        1300   ops/us
- * getOrCreate_concurrentHashMap          1108   ops/us
- * getOrCreate_concurrentSkipListMap       178   ops/us
- * getOrCreate_synchronizedHashMap           9   ops/us
- * }
      
- *
- * 
      Synchronized {@code HashMap} collapses by ~67% (30/28 to 9 ops/us), the same magnitude seen in
- * {@link ThreadSafeMapD1Benchmark} -- pollution dominates its cost far more than lock contention.
- * {@code ConcurrentHashMap} rises ~38-44% over the unpolluted table (777→1072, 769→1108) with tight
- * error bars (under 8% of the mean) -- a real effect, not noise; the {@link Key2} allocation this
- * benchmark forces on every {@code ConcurrentHashMap} lookup apparently costs relatively less once
- * the JIT already treats the surrounding dispatch as megamorphic. {@code Support} and {@code
- * ConcurrentHashtable} are still the two fastest and remain close to each other, but neither can be
- * called ahead here: their error bars are enormous (±520 and ±591, roughly 35-45% of their own
- * means) and their relative order flips between {@code get} and {@code getOrCreate} -- read both as
- * noise, not as a real reordering. {@code ConcurrentSkipListMap} rises modestly, also within a wide
- * error bar -- directional only.
  */
 @Fork(2)
 @Warmup(iterations = 2)
@@ -214,7 +187,6 @@ public static class SharedState {
 
     @Setup(Level.Iteration)
     public void setUp() {
-      BenchmarkUtils.polluteHashDispatch();
       table = ConcurrentHashtable.D2.createBounded(D2Entry.class, CAPACITY);
       supportBuckets = ConcurrentHashtable.createFixedBuckets(SupportEntry.class, CAPACITY);
       concurrentHashMap = new ConcurrentHashMap<>(CAPACITY);
@@ -241,13 +213,12 @@ public void setUp() {
   public static class ThreadState {
     int cursor;
 
-    // Re-pollute every invocation: a one-shot Level.Iteration call gets drowned out by this
-    // benchmark's own real-key traffic well before HotSpot compiles the shared hash dispatch call
-    // sites, letting them re-specialize to a dominant receiver (see
-    // BenchmarkUtils#polluteHashDispatch).
-    @Setup(Level.Invocation)
-    public void pollute() {
-      BenchmarkUtils.polluteHashDispatch();
+    // Front-load pollution once per trial, entirely before JMH's warmup starts: JMH
+    // injects the Blackhole straight into this setup method, so no per-benchmark
+    // scratch state is needed.
+    @Setup(Level.Trial)
+    public void warmUpPollution(Blackhole bh) {
+      BenchmarkUtils.warmUpHashDispatch(bh);
     }
 
     int next() {

From 165dfc0d86e40629df6462504bbed8a69a7cc4fb Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Fri, 25 Sep 2026 00:16:48 -0400
Subject: [PATCH 29/52] Correct benchmark dispatch claims and measure
 allocation directly

Several benchmark javadocs attributed cost to "megamorphic hashCode()/equals()
dispatch" that could not have occurred. A megamorphic profile does not always
imply virtual dispatch: an exact static type is a proof that outranks the
profile, and these keys are statically exact, so C2 devirtualizes without
consulting the polluted profile at all.

- Add polluteComparatorDispatch so Comparator-based TreeSet/TreeMap/
  ConcurrentSkipListMap sites are exercised alongside the natural-ordering ones,
  which do not share their call sites.
- Document the devirtualization model once, in ImmutableMapBenchmark, instead of
  re-deriving it per file.
- Drop the unsupported megamorphic-dispatch attributions in HashtableD1,
  HashtableD2 and TagMapAccess. String is final and Key2 is a final class built
  at the call site, so both sharpen to an exact type.
- Rewrite TagMapAccess's rerun section against same-JDK Zulu 17 data rather than
  a cross-JDK comparison, removing that confound entirely.
- Replace Objects.hash with HashingUtils.hash in Key2. Objects.hash is varargs
  and allocated an Object[] per lookup, handicapping the HashMap baseline by
  24 B/op.
- Fold measured -prof gc results into HashtableD1 and HashtableD2: 24 B/op for
  D1's boxed Long against zero for the hashtable, and 48 B/op for D2 (Long plus
  Key2), confirming escape analysis does not eliminate the composite key.

HashtableD2's throughput tables still need refreshing from a multi-fork run,
since the Key2 change alters that baseline.

Co-Authored-By: Claude Opus 5 (1M context) 
---
 .../trace/api/TagMapAccessBenchmark.java      |  52 ++++---
 .../datadog/trace/util/BenchmarkUtils.java    | 143 +++++++++++-------
 .../trace/util/HashtableD1Benchmark.java      |  34 +++--
 .../trace/util/HashtableD2Benchmark.java      |  42 +++--
 .../trace/util/ImmutableMapBenchmark.java     |   9 +-
 .../trace/util/ImmutableSetBenchmark.java     |  13 +-
 .../util/ThreadSafeMapCounterBenchmark.java   |  14 +-
 7 files changed, 194 insertions(+), 113 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java
index 377d528cbe6..dae425c6b66 100644
--- a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java
@@ -58,30 +58,44 @@
  * TagMapAccessBenchmark.insert_via_ledger            thrpt    5  41169656.095 ±  773264.754  ops/s
  * 
  *
- * 
      Rerun on JDK 8 with a new top-level {@code @Setup(Level.Trial)} calling {@link
- * BenchmarkUtils#warmUpHashDispatch} (this file had none before). M ops/s, 8 threads:
+ * 
      Rerun on the same machine and JDK (Zulu 17.0.7, {@code @Fork(2)}, {@code @Threads(8)}) with a
+ * new top-level {@code @Setup(Level.Trial)} calling {@link BenchmarkUtils#warmUpHashDispatch} (this
+ * file had none before):
  *
  * 
      {@code
- * getEntry                        83   getObject                    87
- * insert                          37   insert_hashMap                48
- * insert_hashMap_builderStyle     20   insert_via_ledger             37*
+ * Benchmark                      Score (M ops/s)    Error
+ * getEntry                              97.00      ± 2.37
+ * getObject                             97.70      ± 0.88
+ * insert                                52.67      ± 1.28
+ * insert_hashMap                        69.57      ± 0.65
+ * insert_hashMap_builderStyle           29.51      ± 0.50
+ * insert_via_ledger                     37.15      ± 0.78
  * }
      
  *
- * 
      * = error bar about a quarter of the mean at {@code @Fork(2)} — directional only.
+ * 
      Pollution left these paths materially unchanged, but not for the same reason on both sides.
+ * {@code TagMap} is typed to {@code String} keys throughout ({@link TagMap#getEntry(String)},
+ * {@link TagMap.Ledger#set(String, Object)}) and compares against a {@code String}-declared field,
+ * so it has no {@code Object}-typed dispatch site to pollute — it is insulated by construction,
+ * whatever the JIT decides. {@code HashMap} does have such sites: {@code getNode} calls {@code
+ * hashCode()}/{@code equals()} on an {@code Object}-declared key. Erasure makes that one bytecode
+ * index serve every map in the application, so in production it sees many receiver types and is
+ * genuinely megamorphic — which is the condition {@code warmUpHashDispatch} reproduces here. Even
+ * so, {@code get}/{@code put} inline into a caller holding a statically exact {@code String}, so C2
+ * sharpens the argument and devirtualizes without consulting that profile. That is a realistic
+ * condition rather than an artifact — plenty of production call sites do hand HashMap a statically
+ * known key type, and it legitimately gets that benefit. Where the key type is not deducible, the
+ * megamorphic profile governs HashMap and still does not reach TagMap.
  *
- * 
      Every number here is 9-29% below the Java 17 table above, with no clear split between the
- * TagMap paths and the HashMap paths this pollution should affect. The table above is Java 17; this
- * rerun is JDK 8, whose C2 backend for Apple Silicon (AArch64) is far less mature than JDK 17+'s —
- * a broad-based slowdown across every entry is expected from that JDK gap alone, independent of
- * pollution — the same JDK-crossing explanation applies to {@link
- * datadog.trace.util.CaseInsensitiveMapBenchmark}'s rerun. ({@link
- * datadog.trace.util.HashtableD1Benchmark} and {@link datadog.trace.util.HashtableD2Benchmark} saw
- * a similar broad drop despite holding the JDK constant — that one is same-session run-to-run
- * noise, not a JDK effect.) The relative story survives: {@code insert_hashMap} (48M) still beats
- * {@code insert} (37M) for plain insertion, and {@code insert_via_ledger} (37M) still clearly beats
- * the HashMap builder-style path (20M); {@code insert_via_ledger} landing roughly level with {@code
- * insert} here (vs. clearly behind it in the table above) is within that path's own wide error bar,
- * not a new finding.
+ * 
      Consistent with that, most entries move by a couple of percent with overlapping intervals, and
+ * the larger moves go up, which pollution cannot cause.
+ *
+ * 
      The exception is {@code insert_via_ledger}, down about 10% (41.17 to 37.15) with
+ * non-overlapping intervals. That is the most allocation-heavy path measured, and {@code
+ * warmUpHashDispatch} itself allocates heavily before measurement starts, so setup churn shifting
+ * GC state for the trial is a likelier cause than dispatch. Not investigated further.
+ *
+ * 
      Both tables are the same machine and JDK, but the baseline above ran at {@code Cnt 5} against
+ * {@code Cnt 10} here, and on a different day, so small differences carry no weight.
  */
 @BenchmarkMode(Mode.Throughput)
 @OutputTimeUnit(TimeUnit.SECONDS)
diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index 6b8cdf9e4b8..9dd6cbbbd73 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -1,6 +1,7 @@
 package datadog.trace.util;
 
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -12,7 +13,17 @@
 import org.openjdk.jmh.annotations.CompilerControl;
 import org.openjdk.jmh.infra.Blackhole;
 
-/** Shared setup helpers for JMH benchmarks in this module. */
+/**
+ * Shared setup helpers for JMH benchmarks in this module.
+ *
+ * 
      {@link Blackhole} solves one correctness problem: it stops the JIT from proving a result is
+ * dead and eliminating the code that produced it. It says nothing about a second, separate problem
+ * -- whether a call site's receiver-type profile still looks like production once measurement
+ * starts. A benchmark can consume every result through a {@code Blackhole} and still measure a
+ * devirtualized, artificially monomorphic fast path that never occurs in the real system. The
+ * pollution helpers here ({@link #warmUpHashDispatch}) exist to guard against that second problem;
+ * they're not redundant with {@code Blackhole}, they cover the axis it doesn't.
+ */
 public final class BenchmarkUtils {
   private BenchmarkUtils() {}
 
@@ -36,55 +47,37 @@ private BenchmarkUtils() {}
   private static final int WARM_UP_ITERATIONS = 50_000;
 
   /**
-   * Call once from {@code @Setup(Level.Trial)}, passing the {@link Blackhole} JMH injects into the
-   * setup method. Exercises {@link HashSet}/{@link java.util.HashMap}, the tracer's {@link
-   * CollectionUtils#tryMakeImmutableSet} immutable sets, {@link ConcurrentHashMap}, and {@link
-   * TreeMap}/{@link TreeSet}/{@link ConcurrentSkipListMap} with several distinct key classes, so
-   * each structure's internal {@code hashCode()}/{@code equals()}/{@code compareTo()} dispatch -- a
-   * call site shared JVM-wide by every instance of that structure in the process, regardless of
-   * which specific instance or call site invokes {@code add}/{@code contains}/{@code get} -- is
-   * already megamorphic before a benchmark measures lookups against a single key type.
+   * Exercises shared collection methods with several key classes before benchmark warmup.
    *
-   * 
      This matches production: those shared internal call sites are hit by every hash- or
-   * sorted-based structure in the JVM across whatever key types the whole application uses, so
-   * they're realistically almost always megamorphic. An isolated benchmark that only ever looks up
-   * one key type (e.g. {@code String}) would otherwise leave them artificially monomorphic for the
-   * entire run, understating real dispatch cost.
+   * 
      HotSpot records receiver types at bytecode call sites, not per collection instance. Scratch
+   * collections therefore contribute to the internal profiles used when compiling benchmark
+   * lookups. Loading extra classes alone can defeat class-hierarchy analysis (optimization based on
+   * the loaded implementations), but changing a receiver profile requires method calls.
    *
-   * 
      Deliberately does not touch the benchmark's own {@code contains}/{@code add}/{@code get}
-   * call sites -- those are realistically free to specialize per caller, the way a genuinely hot,
-   * narrowly-typed call site would in production.
+   * 
      {@link HashSet} uses {@link HashMap}'s lookup code, also used by {@code LinkedHashMap}.
+   * {@link ConcurrentHashMap}, including its key-set views, has separate internal call sites. The
+   * immutable set and map implementations selected by {@link CollectionUtils} are exercised
+   * separately too; on older JDKs these helpers fall back to mutable collections.
    *
-   * 
      Not to be confused with the CHA-defeat decoys in {@code SingleThreadedMapBenchmark}/{@code
-   * ThreadSafeMapBenchmark} ({@code KeyStrategy} implementors referenced only so they're loaded,
-   * never invoked): that technique denies class-hierarchy analysis a single-implementor bet for a
-   * narrow, dd-trace-java-owned interface, and works by class-loading alone. It doesn't apply here
-   * -- {@code Object.hashCode()}/{@code equals()}/{@code compareTo()} already have countless
-   * implementors loaded in any real JVM, so a single-implementor CHA bet was never available for
-   * them. What gates their dispatch is the interpreter's per-call-site type profile, which only
-   * invocation can pollute -- hence this helper actually calls {@code add}/{@code contains}/{@code
-   * get}, rather than just loading classes.
+   * 
      {@link #polluteCompareToDispatch(Blackhole)} also exercises natural-order lookups in {@link
+   * TreeSet}, {@link TreeMap}, and {@link ConcurrentSkipListMap}; {@link
+   * #polluteComparatorDispatch(Blackhole)} exercises the same three collections' separate {@link
+   * Comparator}-based call sites.
    *
-   * 
      {@code @Setup(Level.Trial)}, not {@code Level.Invocation}: the latter's cost is included in
-   * every {@code Throughput}/{@code AverageTime} measurement's own timed window (JMH has no way to
-   * subtract per-invocation setup cost without adding per-op {@code System.nanoTime()} overhead of
-   * its own), so once pollution isn't free relative to the benchmarked op, it would dominate the
-   * reported number instead of the thing being measured.
-   *
-   * 
      Every collection here is freshly allocated per pass and immediately consumed via {@code bh}
-   * -- not just the {@code boolean}/lookup results, but the collection instances themselves.
-   * Consuming only a lookup's result would leave the freshly-allocated, never-escaping collection
-   * open to scalar replacement: once HotSpot compiles this loop as its own unit and proves a
-   * just-allocated {@code HashSet} never escapes it, escape analysis can devirtualize the {@code
-   * hashCode()}/{@code equals()} calls against that specific instance directly, bypassing the
-   * shared, megamorphic call site entirely. Forcing every collection itself through the {@link
-   * Blackhole} closes that loophole, so the allocations here don't need to be reused across calls
-   * the way a measured hot path would.
+   * 
      The benchmark's own collection call sites are not invoked here and remain free to specialize
+   * for their receiver types. A call site's recorded type profile isn't threatened by which key
+   * type happens to dominate traffic during or after warmup -- once HotSpot has recorded multiple
+   * receiver types there, it stays megamorphic regardless of later call frequency. The risk this
+   * class guards against is a statically deducible receiver type bypassing the profile entirely:
+   * loading extra classes here defeats class-hierarchy analysis (optimization based on which
+   * implementations are actually loaded), and {@link #distinctEqualCopy}'s {@code DONT_INLINE}
+   * stops the JIT from tracing a decoy key's type back to its origin through static inference.
    */
   public static void warmUpHashDispatch(Blackhole bh) {
     for (int i = 0; i < WARM_UP_ITERATIONS; ++i) {
       polluteHashDispatch(bh);
       polluteCompareToDispatch(bh);
+      polluteComparatorDispatch(bh);
     }
   }
 
@@ -115,23 +108,16 @@ private static void polluteHashDispatch(Blackhole bh) {
   }
 
   /**
-   * Counterpart to {@link #polluteHashDispatch} for the {@code compareTo}-based dispatch used by
-   * {@code TreeMap}/{@code TreeSet}/{@code ConcurrentSkipListMap}, instead of {@code hashCode()}/
-   * {@code equals()}.
+   * Exercises natural-order {@code compareTo} calls in {@link TreeSet}, {@link TreeMap}, and {@link
+   * ConcurrentSkipListMap} with the default decoy keys.
    *
-   * 
      Unlike the hash-based structures above, a single sorted collection can't hold multiple decoy
-   * key classes at once: natural ordering calls {@code key.compareTo(existing)}, which throws
-   * {@code ClassCastException} the moment two mutually-incomparable types meet (e.g. a {@code
-   * String} and an {@code Integer}). So each key type gets its own fresh collection here, one type
-   * at a time. That's still enough to make the dispatch megamorphic: HotSpot's type profile lives
-   * on the bytecode call site inside {@code TreeMap}/{@code TreeSet}/{@code
-   * ConcurrentSkipListMap}'s shared implementation, not on any one collection instance, so driving
-   * several receiver types through that site across several collection instances pollutes it
-   * exactly as effectively as driving them through one shared instance would.
+   * 
      Each key type uses a separate collection because the decoys are not mutually comparable. The
+   * instances still execute the same internal call sites and contribute to their receiver profiles.
+   * {@link #polluteComparatorDispatch(Blackhole)} is the counterpart for the separate call sites
+   * these collections use when constructed with an explicit {@link java.util.Comparator}.
    *
-   * 
      {@code Object}'s decoy is skipped: it isn't {@link Comparable}, so it has no natural
-   * ordering to dispatch through in the first place -- the same reason it's exempt from an {@code
-   * equals()}-identity copy in {@link #distinctEqualCopy}.
+   * 
      Keys that do not implement {@link Comparable}, including the plain {@code Object} decoy, are
+   * skipped.
    */
   private static void polluteCompareToDispatch(Blackhole bh) {
     for (Object key : DECOY_KEYS) {
@@ -156,6 +142,45 @@ private static void polluteCompareToDispatch(Blackhole bh) {
     }
   }
 
+  /**
+   * Exercises {@link Comparator}-based {@code compare} calls in {@link TreeSet}, {@link TreeMap},
+   * and {@link ConcurrentSkipListMap} with the default decoy keys.
+   *
+   * 
      Constructing these collections with an explicit {@code Comparator} routes lookups through
+   * internal call sites distinct from the no-arg, natural-ordering constructors {@link
+   * #polluteCompareToDispatch(Blackhole)} exercises -- pollution there does not carry over here.
+   * The comparator itself delegates to {@link Comparable#compareTo}, so this pass also keeps that
+   * method's call site (invoked from inside the comparator, not from the collection directly) warm
+   * across the same key types.
+   *
+   * 
      Keys that do not implement {@link Comparable}, including the plain {@code Object} decoy, are
+   * skipped.
+   */
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  private static void polluteComparatorDispatch(Blackhole bh) {
+    Comparator comparator = (a, b) -> ((Comparable) a).compareTo(b);
+    for (Object key : DECOY_KEYS) {
+      if (!(key instanceof Comparable)) {
+        continue;
+      }
+
+      TreeSet treeSet = new TreeSet<>(comparator);
+      treeSet.add(key);
+      bh.consume(treeSet.contains(distinctEqualCopy(key)));
+      bh.consume(treeSet);
+
+      TreeMap treeMap = new TreeMap<>(comparator);
+      treeMap.put(key, key);
+      bh.consume(treeMap.get(distinctEqualCopy(key)));
+      bh.consume(treeMap);
+
+      ConcurrentSkipListMap skipListMap = new ConcurrentSkipListMap<>(comparator);
+      skipListMap.put(key, key);
+      bh.consume(skipListMap.get(distinctEqualCopy(key)));
+      bh.consume(skipListMap);
+    }
+  }
+
   /**
    * Returns a distinct instance that's {@code .equals()} to {@code key} but never {@code ==} it, so
    * the lookup that follows can't take {@code HashMap}/{@code ConcurrentHashMap}'s internal {@code
@@ -171,7 +196,11 @@ private static void polluteCompareToDispatch(Blackhole bh) {
    * and devirtualize {@code hashCode()}/{@code equals()} via static type inference alone --
    * bypassing the shared, runtime type profile this class exists to pollute, no matter how many
    * distinct instances or types are pushed through it. Keeping this a real, non-inlined call forces
-   * every caller to go through actual dispatch.
+   * every caller to go through actual dispatch -- a type-erasing wormhole, the mirror image of
+   * {@link Blackhole}'s value-erasing one.
+   *
+   * @param key the decoy key to copy
+   * @return a distinct-but-equal copy, or {@code key} itself if it has no distinct-but-equal form
    */
   @CompilerControl(CompilerControl.Mode.DONT_INLINE)
   @SuppressWarnings(
diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
index 28998f7ef48..1b9731cda0b 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
@@ -39,13 +39,17 @@
  * 
  *
  * 
      Update is where Hashtable dominates: D1 is ~14x faster on JDK 8 (see the Java 17 rerun
- * below for a narrower but still decisive margin), because the HashMap path allocates per call (a
- * {@code Long}) and the resulting GC pressure throttles throughput under multiple threads. This is
- * the headline case for {@code Hashtable}: a simple counter/tally with a primitive value is exactly
- * where HashMap's autoboxing tax bites hardest, and {@code Hashtable.D1} sidesteps it entirely by
- * mutating a field on the retrieved entry in place. Add is roughly comparable (both allocate
- * one entry per insert). Iterate is essentially a wash on JDK 8, though not on Java 17 (see
- * below). 
+ * below for a narrower but still decisive margin). D1 mutates a primitive counter in the existing
+ * entry; the HashMap path boxes a {@code Long} on every {@code merge}. Measured with {@code -prof
+ * gc} on Zulu 17, {@code update_hashMap} allocates 24.000 ± 0.001 B/op — exactly one boxed {@code
+ * Long} (12-byte header plus an 8-byte value, aligned to 24) — against ≈0 B/op for {@code
+ * update_hashtable}, and 386 collections over the run against none. The GC pressure is measured
+ * rather than inferred from throughput. This is the headline case for {@code Hashtable}: a simple
+ * counter/tally with a primitive value is exactly where HashMap's autoboxing tax bites hardest, and
+ * {@code Hashtable.D1} sidesteps it entirely by mutating a field on the retrieved entry in place.
+ * Add is roughly comparable — both allocate one entry per insert, and the error bars exceed
+ * the means, so no precise comparison is possible there. Iterate is essentially a wash on
+ * JDK 8, though not on Java 17 (see below). 
  * MacBook M1 8 threads (Java 8)
  *
  * Benchmark                                Mode  Cnt     Score     Error   Units
@@ -66,13 +70,15 @@
  * Hashtable.D1.Entry#matches} are call sites private to {@code Hashtable.java}, structurally
  * distinct from {@code java.util.HashMap}/{@code HashSet}'s internal {@code hashCode()}/{@code
  * equals()} call sites — JIT type profiles are keyed per call site, so {@code warmUpHashDispatch}
- * cannot reach them regardless of key-type overlap. Since the JDK and machine were held constant
- * across this rerun (unlike the JDK 8-vs-17 comparisons in {@link
- * datadog.trace.util.CaseInsensitiveMapBenchmark} and {@link
- * datadog.trace.api.TagMapAccessBenchmark}), the drop here is same-session run-to-run noise
- * (thermal/power, not controlled for) rather than either a pollution effect or a JDK effect. The
- * relative conclusion (D1 dominates {@code update}, is roughly comparable on {@code add},
- * ties on {@code iterate}) is unchanged either way.
+ * cannot reach them regardless of key-type overlap. It is equally a no-op for {@code *_hashMap}:
+ * the keys come from {@code SOURCE_KEYS}, a {@code String[]}, and {@code String} is final, so C2
+ * sharpens the {@code Object}-declared key to an exact type and devirtualizes {@code
+ * hashCode()}/{@code equals()} without consulting the polluted profile. Pollution therefore cannot
+ * explain a drop on either side. The JDK and machine were held constant, so what remains is
+ * uncontrolled run-to-run variation plus one concrete candidate: {@code warmUpHashDispatch} itself
+ * allocates heavily before measurement starts, which can shift GC state for the whole trial.
+ * Neither was measured. The relative conclusion (D1 dominates {@code update}, is roughly
+ * comparable on {@code add}, ties on {@code iterate}) is unchanged either way.
  *
  * 
      Separately rerun on Zulu 17.0.7 (native AArch64, same machine, pollution wiring unchanged; JMH
  * auto-detected the cheap "compiler" Blackhole mode here, unlike JDK 8, so absolute numbers below
diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
index ae8510890ba..94067901d1d 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
@@ -43,12 +43,26 @@
  * (Java has no built-in tuple-as-key) — D2 sidesteps it by taking both key parts directly.
  *
  * 
      Update is where Hashtable dominates: D2 is ~26x faster on JDK 8 (see the Java 17 rerun
- * below for a narrower but still decisive margin), because the HashMap path allocates per call (a
- * {@code Long}, plus a {@code Key2}) and the resulting GC pressure throttles throughput under
- * multiple threads. Like D1, this is the headline case for {@code Hashtable}: a simple
- * counter/tally with a primitive value is exactly where HashMap's autoboxing tax bites hardest.
- * Add is ~3x faster for D2 (Hashtable sidesteps the {@code Key2} allocation). Iterate
- * is essentially a wash on JDK 8, though not on Java 17 (see below). 
+ * below for a narrower but still decisive margin), because the HashMap path allocates per call.
+ * Measured with {@code -prof gc} on Zulu 17, {@code update_hashMap} allocates 48.000 ± 0.001 B/op,
+ * which decomposes exactly: 24 for the boxed {@code Long} and 24 for the {@code Key2} wrapper
+ * (12-byte header, two references, one {@code int}). {@code update_hashtable} allocates ≈0 B/op.
+ * Collections over the run were 420 against none.
+ *
+ * 
      That 48 also answers a common assumption: escape analysis does not eliminate the
+ * temporary {@code Key2}, even though every key in this benchmark is already present and the
+ * wrapper is discarded immediately. C2 assigns one escape state per allocation site rather than per
+ * path, and {@code merge} stores the key into a {@code Node} on the absent branch, so the
+ * allocation is GlobalEscape for the whole compiled method. Eliminating it would require
+ * specializing the present and absent cases separately, which HotSpot does not do. (The wrapper's
+ * hash uses {@link HashingUtils#hash(Object, Object)} rather than {@code Objects.hash}, whose
+ * varargs array would otherwise add a third 24-byte allocation per lookup and handicap the
+ * baseline.)
+ *
+ * 
      Like D1, this is the headline case for {@code Hashtable}: a simple counter/tally with a
+ * primitive value is exactly where HashMap's autoboxing tax bites hardest. Add is ~3x faster
+ * for D2 (Hashtable sidesteps the {@code Key2} allocation). Iterate is essentially a wash on
+ * JDK 8, though not on Java 17 (see below). 
  * MacBook M1 8 threads (Java 8)
  *
  * Benchmark                                Mode  Cnt     Score     Error   Units
@@ -70,11 +84,15 @@
  * be a no-op here: {@link Hashtable.D2.Entry#hash} and {@link Hashtable.D2.Entry#matches} are call
  * sites private to {@code Hashtable.java}, structurally distinct from {@code
  * java.util.HashMap}/{@code HashSet}'s internal dispatch call sites — pollution cannot reach them.
- * With the JDK and machine held constant across this rerun, the drop is same-session run-to-run
- * noise (thermal/power, not controlled for) rather than a genuine pollution effect. Treat these two
- * runs as not directly comparable on absolute numbers. The relative conclusion (D2 dominates
- * {@code update}, wins {@code add} by avoiding the {@code Key2} allocation, ties on {@code
- * iterate}) is unchanged either way.
+ * It is equally a no-op for {@code *_hashMap}: {@code Key2} is a final class and every lookup key
+ * is built at the call site with {@code new Key2(...)}, so C2 has an exact type either way and
+ * devirtualizes {@code hashCode()}/{@code equals()} without consulting the polluted profile.
+ * Pollution therefore cannot explain a move on either side. With the JDK and machine held constant,
+ * what remains is uncontrolled run-to-run variation plus one concrete candidate: {@code
+ * warmUpHashDispatch} itself allocates heavily before measurement starts, which can shift GC state
+ * for the whole trial. Neither was measured. Treat these two runs as not directly comparable on
+ * absolute numbers. The relative conclusion (D2 dominates {@code update}, wins {@code add}
+ * by avoiding the {@code Key2} allocation, ties on {@code iterate}) is unchanged either way.
  *
  * 
      Separately rerun on Zulu 17.0.7 (native AArch64, same machine, pollution wiring unchanged).
  * JMH auto-detected the cheap "compiler" Blackhole mode on Java 17 (its log explicitly warns that
@@ -140,7 +158,7 @@ static final class Key2 {
     Key2(String k1, Integer k2) {
       this.k1 = k1;
       this.k2 = k2;
-      this.hash = Objects.hash(k1, k2);
+      this.hash = HashingUtils.hash(k1, k2);
     }
 
     @Override
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
index c64826f7176..58edce8873c 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
@@ -50,8 +50,13 @@
  *
  * 
      A virtual call site is monomorphic when it has observed one receiver class,
  * polymorphic when it has observed a small set, and megamorphic when no small, stable
- * set dominates. HotSpot can usually devirtualize and inline the monomorphic case, sometimes a
- * small polymorphic one; megamorphic sites generally retain virtual dispatch.
+ * set dominates. HotSpot keeps one such profile per bytecode index, shared across every inlining
+ * context, and consults it as a fallback: the profile says what to speculate when the receiver type
+ * cannot be deduced. A megamorphic profile therefore does not always imply virtual dispatch in
+ * compiled code. Wherever a particular context yields a proof of the receiver type — a final class,
+ * an exact type from an allocation or a constant, or an argument sharpened by inlining — C2
+ * devirtualizes and inlines regardless of how polluted the profile is. Pollution only reaches the
+ * contexts that have no such proof.
  *
  * 
      Java 17 results on an Apple M1 with the front-loaded {@link BenchmarkUtils#warmUpHashDispatch}
  * pollution design, {@code @Fork(5)}, {@code @Threads(8)} (M ops/s):
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java
index 6e9bccf5398..fcc2f5209cd 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java
@@ -48,13 +48,16 @@
  * 
      Lookup variants:
  *
  * 
        
- *   
      • {@code hit} uses the same interned strings that were inserted, exercising the identity fast
- *       path.
- *   
      • {@code hitFresh} uses equal, non-interned strings, avoiding the identity fast path. It is
- *       measured only for the hash-based structures.
- *   
      • {@code miss} uses non-interned strings that are not in the set.
+ *   
      • {@code hit} reuses the inserted interned strings, exercising identity fast paths where
+ *       available.
+ *   
      • {@code hitFresh} reuses equal, non-interned copies created before measurement, avoiding
+ *       identity matches without allocating per lookup; only hash-based structures have this case.
+ *   
      • {@code miss} reuses non-interned strings that are not in the set.
  * 
      
  *
+ * 
      For hash-based lookups, warmup populates the reused strings' cached hashes; these cases do not
+ * measure repeated string hashing from characters.
+ *
  * 
      Java 17 results on an Apple M1 with the front-loaded {@link BenchmarkUtils#warmUpHashDispatch}
  * pollution design, {@code @Fork(5)}, {@code @Threads(8)} (M ops/s):
  *
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java
index 0dbbdb3f5b9..0449d30c54d 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapCounterBenchmark.java
@@ -56,7 +56,13 @@
  * 
  *
  * 
      Rerun with {@link BenchmarkUtils#warmUpHashDispatch} wired into {@code SharedState.setUp()},
- * same JDK 17 as the table above -- a clean pollution-only delta:
+ * same JDK 17 and machine as the table above. Same-JDK removes one variable, but separate JMH
+ * invocations aren't a controlled A/B -- forks of a single benchmark method run back-to-back, while
+ * the two tables here come from separate {@code ./gradlew jmh} invocations, so a systematic
+ * difference between them (thermal state, background load, where the JIT happened to land) isn't
+ * distinguishable from a pollution effect. Pollution itself is best-effort -- it raises the odds a
+ * shared call site is megamorphic going into measurement, not a guarantee -- so treat this rerun as
+ * illustrative, not as an isolated measurement of the pollution mechanism's effect:
  *
  * 
      {@code
  * Benchmark                          Score   Units
@@ -67,9 +73,9 @@
  *
  * 
      {@code ConcurrentHashtable} and {@code AtomicLong} are still within 6% of each other (68 vs 72
  * ops/us), unchanged from above. {@code LongAdder}'s score jumped to 205 ops/us, but its error bar
- * ({@code ±429}) is more than double its own mean -- unusable at this fork count, not evidence of a
- * real pollution effect; take the "within 15%" finding above as still the reliable read for {@code
- * LongAdder} too.
+ * ({@code ±429}) is more than double its own mean -- unusable at this fork count, and not evidence
+ * of a real pollution effect. It equally cannot confirm the earlier within-15% comparison: that
+ * finding rests on the first table's own data, and this run neither supports nor refutes it.
  */
 @Fork(2)
 @Warmup(iterations = 2)

From 38783798263d8fbe8349b25b7b42f4275296202e Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Fri, 25 Sep 2026 00:25:35 -0400
Subject: [PATCH 30/52] Correct the Object decoy's contribution in
 distinctEqualCopy javadoc

An identity match yields the same lookup result but never invokes equals(), so
the Object decoy contributes a hashCode() receiver sample and no equals() one.
The previous wording called the two indistinguishable, which is backwards for
the profiling purpose this class exists to serve.

Co-Authored-By: Claude Opus 5 (1M context) 
---
 .../src/jmh/java/datadog/trace/util/BenchmarkUtils.java    | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index 9dd6cbbbd73..50f786d5a44 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -186,9 +186,10 @@ private static void polluteComparatorDispatch(Blackhole bh) {
    * the lookup that follows can't take {@code HashMap}/{@code ConcurrentHashMap}'s internal {@code
    * key == storedKey || key.equals(storedKey)} identity fast path and skip calling {@code equals()}
    * -- which is exactly the dispatch this class exists to pollute. {@code Object}'s own {@code
-   * equals()} is identity, so a decoy of that type has no distinct-but-equal instance to make; it's
-   * returned as-is, and the identity fast path is then indistinguishable from a genuine {@code
-   * equals()} call anyway.
+   * equals()} is identity, so a decoy of that type has no distinct-but-equal instance to make and
+   * is returned as-is. The lookup then matches on identity: same result, but {@code equals()} is
+   * never invoked, so the {@code Object} decoy contributes a {@code hashCode()} receiver sample
+   * (the hash is computed before the identity check) and no {@code equals()} one.
    *
    * 
      {@code DONT_INLINE} makes this call boundary an optimization black box: without it, once a
    * caller like {@link #polluteHashDispatch} is inlined into a tight loop, the JIT can trace a

From 99ca3515519594e0105e8887318cb11ce73c7780 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Fri, 25 Sep 2026 00:31:19 -0400
Subject: [PATCH 31/52] Attribute the synchronized HashMap gap to lock
 contention, not dispatch

Both ThreadSafeMap benchmarks claimed that under pollution, megamorphic
hashCode()/equals() dispatch dominated the synchronized HashMap cost "far more
than lock contention" -- which also asserted that each benchmark failed to
measure the thing it was built to measure.

The arithmetic rules that out: get_synchronizedHashMap runs at 9.2 ops/us (D1)
and 8.8 (D2), roughly 110 ns per op aggregate across eight threads. A
megamorphic dispatch costs single-digit nanoseconds; a contended monitor with
eight threads parking and unparking does not. Separately, the dispatch is not
megamorphic in compiled code anyway: D1 keys are String, a final class, and D2's
Key2 is a final class built at the call site, so C2 sharpens to an exact type and
devirtualizes without consulting the profile.

Co-Authored-By: Claude Opus 5 (1M context) 
---
 .../java/datadog/trace/util/ThreadSafeMapD1Benchmark.java | 8 +++++---
 .../java/datadog/trace/util/ThreadSafeMapD2Benchmark.java | 8 +++++---
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
index 600cc5152df..bc5c9efb6f2 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
@@ -60,9 +60,11 @@
  *       path.
  *   
    • {@code ConcurrentSkipListMap} is ~6× slower than {@code ConcurrentHashMap} — tree traversal
  *       cost is high even under lock-free CAS.
- *   
    • Synchronized {@code HashMap} is over 150× slower than {@code ConcurrentHashtable}; under
- *       pollution its megamorphic {@code hashCode()}/{@code equals()} dispatch dominates its cost
- *       far more than the lock contention this benchmark was designed to isolate.
+ *   
    • Synchronized {@code HashMap} is over 150× slower than {@code ConcurrentHashtable} (9.2 vs
+ *       1405.8 ops/us) — lock contention across eight threads on a single monitor, which is what
+ *       this benchmark isolates. Type-profile pollution is not a factor: the keys are {@code
+ *       String}, a final class, so these lookups devirtualize by exact type and never consult the
+ *       polluted profile.
  *   
    • {@code getOrCreate} is near-identical to {@code get} because all keys are pre-populated —
  *       the lock branch is never taken during measurement.
  * 
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
index 15448be5343..2330045dc33 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
@@ -70,9 +70,11 @@
  *       {@code int} K2 field avoids boxing inside the entry match on the write-path re-check.
  *   
    • {@code ConcurrentSkipListMap} is ~6× slower than {@code ConcurrentHashMap} due to tree
  *       traversal.
- *   
    • Synchronized {@code HashMap} is over 150× slower than the fastest options; under pollution
- *       its megamorphic {@code hashCode()}/{@code equals()} dispatch dominates its cost far more
- *       than lock contention, the same magnitude seen in {@link ThreadSafeMapD1Benchmark}.
+ *   
    • Synchronized {@code HashMap} is over 150× slower than the fastest options (8.8 vs 1573.5
+ *       ops/us) — lock contention across eight threads on a single monitor, the same magnitude seen
+ *       in {@link ThreadSafeMapD1Benchmark}. Type-profile pollution is not a factor: {@code Key2}
+ *       is a final class built at the call site, so C2 has an exact type and devirtualizes without
+ *       consulting the polluted profile.
  * 
  */
 @Fork(2)

From 19c00ff3532fab145fb5faad61ba0b1d2b9cb5f2 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Fri, 25 Sep 2026 07:38:58 -0400
Subject: [PATCH 32/52] Refresh benchmark tables from a -prof gc run and
 retract stale claims

Full Zulu 17 run across the four affected classes with -prof gc, so every table
now carries allocation per op alongside throughput.

- HashtableD2: the Objects.hash fix roughly doubled update_hashMap (196.7 to
  463.0), so the update margin is ~5.4x rather than the ~11.6x claimed, and add
  is now a tie (1018.9 vs 1009.6) rather than a ~1.8x hashtable win. Both prior
  figures were artifacts of the varargs allocation in the old Key2 constructor.
- ThreadSafeMapD1/D2: refreshed tables moved substantially, so the synchronized
  HashMap contention ratios are ~280-290x, not the ~150x computed from the older
  numbers a few commits ago.
- ThreadSafeMapD2: document that Key2 is scalar-replaced in every map arm
  (0 B/op), confirmed by LogCompilation showing eliminate_allocation once the
  never-taken computeIfAbsent branch is pruned as unstable_if. Flag that this
  depends on a one-sided branch profile the benchmark manufactures by populating
  in @Setup; a production miss rate would restore the allocation, so those
  numbers are an upper bound.
- HashtableD1: the collection count in prose was from the single-fork spot check
  and disagreed with its own table; now both read 852.

Allocation figures decompose exactly, which is a useful property to keep: 24 B/op
is one boxed Long, 32 is a HashMap.Node, 48 is Long plus Key2, and 56 is Key2
plus Node with no box because (long) i for i < 128 hits the Long.valueOf cache.

Co-Authored-By: Claude Opus 5 (1M context) 
---
 .../trace/util/HashtableD1Benchmark.java      | 36 +++++++----
 .../trace/util/HashtableD2Benchmark.java      | 50 ++++++++++-----
 .../trace/util/ThreadSafeMapD1Benchmark.java  | 32 +++++-----
 .../trace/util/ThreadSafeMapD2Benchmark.java  | 61 ++++++++++++-------
 4 files changed, 117 insertions(+), 62 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
index 1b9731cda0b..645d717c79e 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
@@ -43,7 +43,7 @@
  * entry; the HashMap path boxes a {@code Long} on every {@code merge}. Measured with {@code -prof
  * gc} on Zulu 17, {@code update_hashMap} allocates 24.000 ± 0.001 B/op — exactly one boxed {@code
  * Long} (12-byte header plus an 8-byte value, aligned to 24) — against ≈0 B/op for {@code
- * update_hashtable}, and 386 collections over the run against none. The GC pressure is measured
+ * update_hashtable}, and 852 collections over the run against none. The GC pressure is measured
  * rather than inferred from throughput. This is the headline case for {@code Hashtable}: a simple
  * counter/tally with a primitive value is exactly where HashMap's autoboxing tax bites hardest, and
  * {@code Hashtable.D1} sidesteps it entirely by mutating a field on the retrieved entry in place.
@@ -86,22 +86,34 @@
  * full caveat). ops/us, 8 threads:
  *
  * 
      {@code
- * add_hashMap        1502.6   add_hashtable      1377.3
- * update_hashMap      644.2   update_hashtable   2706.5
- * iterate_hashMap      19.3   iterate_hashtable    78.0
+ * Benchmark            ops/us            B/op   gc.count
+ * add_hashMap        1517.5 ± 242.9      32.0       1820
+ * add_hashtable      1302.1 ± 403.9      40.0       1933
+ * update_hashMap      686.0 ± 140.4      24.0        852
+ * update_hashtable   2770.7 ± 169.4       ~0          ~0
+ * iterate_hashMap      19.8 ±   0.6      40.0         55
+ * iterate_hashtable    79.6 ±   9.6       ~0          ~0
  * }
      
  *
+ * 
      Allocation is measured with {@code -prof gc} and decomposes exactly: 24 B/op for {@code
+ * update_hashMap} is one boxed {@code Long}; 32 B/op for {@code add_hashMap} is one {@code
+ * HashMap.Node}, with no box because {@code (long) i} for {@code i < 128} hits the {@code
+ * Long.valueOf} cache; 40 B/op for {@code add_hashtable} is the {@code D1Counter} entry. The
+ * hashtable's {@code update} and {@code iterate} paths allocate nothing at all, which is the point
+ * of the design.
+ *
  * 
      Within this single run (so the cross-JDK Blackhole-mode confound doesn't apply to the ratios),
- * {@code update_hashtable} still wins by ~4.2x — down from ~14x on JDK 8, because Java 17's
- * allocator/GC absorbs {@code update_hashMap}'s per-call {@code Long} boxing far better than JDK 8
- * did (update_hashMap itself got ~5x faster; update_hashtable only ~1.5x faster). {@code
+ * {@code update_hashtable} wins by ~4.0x — down from ~14x on JDK 8, because Java 17's allocator/GC
+ * absorbs {@code update_hashMap}'s per-call {@code Long} boxing far better than JDK 8 did
+ * (update_hashMap itself got ~5x faster; update_hashtable only ~1.5x faster). {@code
  * iterate_hashtable} also now clearly wins (~4.0x), flipping from JDK 8's "wash" — HashMap's {@code
  * entrySet()} iterator does more per-entry work than a modern JIT's allocation improvements erase.
- * {@code add} is the one case that flips the other way: {@code add_hashMap} edges out {@code
- * add_hashtable} slightly (1502.6 vs 1377.3). Net takeaway: {@code Hashtable} is a strong
- * substitute for {@code HashMap} particularly for simple counter/tally use cases with a primitive
- * value, where avoiding the per-update boxing allocation pays off even on a JVM with much better
- * allocation handling than JDK 8 had.
+ * {@code add} is the one case that flips the other way: {@code add_hashMap} leads on the means
+ * (1517.5 vs 1302.1), though both error bars are wide enough to overlap, so treat that one as
+ * undecided rather than a HashMap win. Net takeaway: {@code Hashtable} is a strong substitute for
+ * {@code HashMap} particularly for simple counter/tally use cases with a primitive value, where
+ * avoiding the per-update boxing allocation pays off even on a JVM with much better allocation
+ * handling than JDK 8 had.
  */
 @Fork(2)
 @Warmup(iterations = 2)
diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
index 94067901d1d..a61d0c86a2e 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
@@ -102,23 +102,43 @@
  * ops/us, 8 threads:
  *
  * 
      {@code
- * add_hashMap         656.7   add_hashtable     1185.5
- * update_hashMap      196.7   update_hashtable  2292.2
- * iterate_hashMap      20.5   iterate_hashtable   69.2
+ * Benchmark            ops/us            B/op   gc.count
+ * add_hashMap        1009.6 ± 190.2      56.0       1819
+ * add_hashtable      1018.9 ± 235.0      40.0       1589
+ * update_hashMap      463.0 ±  95.7      48.0       1064
+ * update_hashtable   2492.7 ±  51.9       ~0          ~0
+ * iterate_hashMap      19.8 ±   0.3      40.0         60
+ * iterate_hashtable    72.0 ±   1.6       ~0          ~0
  * }
      
  *
- * 
      {@code update_hashtable} still wins decisively (~11.6x, down from ~26x on JDK 8 — Java 17's
- * allocator/GC absorbs {@code update_hashMap}'s per-call {@code Long}+{@code Key2} boxing far
- * better than JDK 8 did: update_hashMap got ~3.5x faster, update_hashtable only ~1.6x faster).
- * Unlike JDK 8, Hashtable now wins clearly on every operation: {@code add_hashtable} wins
- * ~1.8x (vs. JDK 8's ~3x — HashMap's {@code Key2} allocation also got relatively cheaper), and
- * {@code iterate_hashtable} flips from JDK 8's wash to a ~3.4x win (HashMap's {@code entrySet()}
- * iterator does more per-entry work than a modern JIT's allocation improvements erase). Net
- * takeaway, consistent with {@link HashtableD1Benchmark}: {@code Hashtable} is a strong substitute
- * for {@code HashMap} particularly for simple counter/tally use cases with a primitive value, where
- * avoiding the per-update boxing allocation pays off even on a JVM with much better allocation
- * handling than JDK 8 had — and for D2 specifically, avoiding the composite-key wrapper allocation
- * pays off across the board, not just on {@code update}.
+ * 
      These numbers postdate the switch from {@code Objects.hash} to {@link
+ * HashingUtils#hash(Object, Object)} in {@code Key2}, which removed a varargs {@code Object[]} per
+ * key construction. Earlier tables in this file predate it and had the HashMap baseline carrying
+ * that extra 24 B/op.
+ *
+ * 
      Allocation decomposes exactly. {@code add_hashMap} at 56 B/op is a {@code Key2} (24) plus a
+ * {@code HashMap.Node} (32), with no box because {@code (long) i} for {@code i < 128} hits the
+ * {@code Long.valueOf} cache. {@code update_hashMap} at 48 B/op is a boxed {@code Long} (24) plus
+ * the {@code Key2} (24) — see the escape-analysis note above. Both hashtable paths that avoid the
+ * wrapper allocate nothing on {@code update} and {@code iterate}.
+ *
+ * 
      {@code update_hashtable} wins by ~5.4x — down from ~26x on JDK 8, and also down from the
+ * ~11.6x this file previously reported. The difference is the {@code Objects.hash} fix: removing
+ * that varargs {@code Object[]} took 24 B/op off {@code update_hashMap} and roughly doubled its
+ * throughput (196.7 to 463.0). {@code iterate_hashtable} wins ~3.6x, flipping JDK 8's wash —
+ * HashMap's {@code entrySet()} iterator does more per-entry work than a modern JIT's allocation
+ * improvements erase.
+ *
+ * 
      {@code add} is now a tie (1018.9 vs 1009.6, comfortably inside both error bars), where this
+ * file previously claimed a ~1.8x hashtable win. That claim was an artifact of the varargs
+ * allocation in the old {@code Key2} constructor; with it gone the two are indistinguishable on the
+ * insert path, which makes sense — both allocate one entry per insert.
+ *
+ * 
      Net takeaway, consistent with {@link HashtableD1Benchmark}: {@code Hashtable} is a strong
+ * substitute for {@code HashMap} for counter/tally use cases with a primitive value, where avoiding
+ * the per-update boxing pays off even on a JVM with much better allocation handling than JDK 8 had.
+ * For D2 specifically, avoiding the composite-key wrapper pays off on {@code update} and {@code
+ * iterate} — but not on {@code add}, contrary to what this file said before the baseline was fixed.
  */
 @Fork(2)
 @Warmup(iterations = 2)
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
index bc5c9efb6f2..d4a1d85d1bb 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
@@ -41,27 +41,31 @@
  * design ({@code @Fork(2)}, {@code @Threads(8)}, 64 pre-populated keys; ops/us):
  *
  * 
      {@code
- * Benchmark                             Score   Units
- * get_concurrentHashtable               1405.8 ops/us
- * get_concurrentHashMap                 1167.3 ops/us
- * get_concurrentSkipListMap              193.3 ops/us
- * get_synchronizedHashMap                  9.2 ops/us
+ * Benchmark                              ops/us          B/op
+ * get_concurrentHashtable            2593.6 ± 116.7        ~0
+ * get_concurrentHashMap              1830.7 ±  31.5        ~0
+ * get_concurrentSkipListMap           202.7 ±  75.6        ~0
+ * get_synchronizedHashMap               9.2 ±   0.8        ~0
  *
- * getOrCreate_concurrentHashtable       1495.2 ops/us
- * getOrCreate_concurrentHashMap         1186.6 ops/us
- * getOrCreate_synchronizedHashMap          8.9 ops/us
+ * getOrCreate_concurrentHashtable    2580.6 ±  28.3        ~0
+ * getOrCreate_concurrentHashMap      1823.9 ±  50.2        ~0
+ * getOrCreate_synchronizedHashMap       9.3 ±   0.7        ~0
  * }
      
  *
+ * 
      Allocation is measured with {@code -prof gc}. Every arm is allocation-free: the keys are
+ * pre-installed {@code String}s reused on each lookup, so nothing is constructed per operation.
+ *
  * 
      Key findings:
  *
  * 
        
- *   
      • {@code ConcurrentHashtable} is ~20% faster than {@code ConcurrentHashMap} on {@code get}
- *       (1405.8 vs 1167.3 ops/us); avoids the hash-to-segment translation CHM pays even on its fast
+ *   
      • {@code ConcurrentHashtable} is ~40% faster than {@code ConcurrentHashMap} on {@code get}
+ *       (2593.6 vs 1830.7 ops/us); avoids the hash-to-segment translation CHM pays even on its fast
  *       path.
- *   
      • {@code ConcurrentSkipListMap} is ~6× slower than {@code ConcurrentHashMap} — tree traversal
- *       cost is high even under lock-free CAS.
- *   
      • Synchronized {@code HashMap} is over 150× slower than {@code ConcurrentHashtable} (9.2 vs
- *       1405.8 ops/us) — lock contention across eight threads on a single monitor, which is what
+ *   
      • {@code ConcurrentSkipListMap} is ~9× slower than {@code ConcurrentHashMap} — tree traversal
+ *       cost is high even under lock-free CAS. Its error bar is wide (±75.6 on a 202.7 mean), so
+ *       treat that multiple as approximate.
+ *   
      • Synchronized {@code HashMap} is roughly 280× slower than {@code ConcurrentHashtable} (9.2
+ *       vs 2593.6 ops/us) — lock contention across eight threads on a single monitor, which is what
  *       this benchmark isolates. Type-profile pollution is not a factor: the keys are {@code
  *       String}, a final class, so these lookups devirtualize by exact type and never consult the
  *       polluted profile.
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
index 2330045dc33..921e4fd4fd2 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
@@ -44,33 +44,52 @@
  * design ({@code @Fork(2)}, {@code @Threads(8)}, 64 pre-populated keys; ops/us):
  *
  * 
        {@code
- * Benchmark                              Score   Units
- * get_support                           1573.5   ops/us
- * get_concurrentHashtable               1412.7   ops/us
- * get_concurrentHashMap                 1055.3   ops/us
- * get_concurrentSkipListMap              172.5   ops/us
- * get_synchronizedHashMap                  8.8   ops/us
+ * Benchmark                               ops/us          B/op
+ * get_support                         2730.0 ±  33.4        ~0
+ * get_concurrentHashtable             2653.0 ±  81.6        ~0
+ * get_concurrentHashMap               1665.0 ±  24.2        ~0
+ * get_concurrentSkipListMap            187.3 ±  40.1        ~0
+ * get_synchronizedHashMap                9.3 ±   0.4        ~0
  *
- * getOrCreate_support                   1492.8   ops/us
- * getOrCreate_concurrentHashtable       1416.7   ops/us
- * getOrCreate_concurrentHashMap         1083.2   ops/us
- * getOrCreate_concurrentSkipListMap      169.9   ops/us
- * getOrCreate_synchronizedHashMap          8.6   ops/us
+ * getOrCreate_support                 2597.4 ± 221.5        ~0
+ * getOrCreate_concurrentHashMap       1647.1 ±  75.5        ~0
+ * getOrCreate_concurrentHashtable     1397.0 ±  46.5        ~0
+ * getOrCreate_concurrentSkipListMap    179.1 ±  40.0        ~0
+ * getOrCreate_synchronizedHashMap        9.3 ±   0.4        ~0
  * }
        
  *
+ * 
        The {@link Key2} wrapper is not allocated in the measured code. Every map arm reports
+ * ≈0 B/op under {@code -prof gc}, despite the source constructing a {@code Key2} per lookup.
+ * LogCompilation confirms the mechanism: C2 emits {@code eliminate_allocation} for {@code Key2},
+ * because the never-taken {@code computeIfAbsent} branch is pruned as {@code unstable_if}, which
+ * removes the only store of the key and leaves it provably non-escaping.
+ *
+ * 
        That is a property of this workload, not of the code, and it would not survive in production.
+ * Pruning is possible only because {@code @Setup} installs every key before warmup, so the absent
+ * branch is never recorded. A real cache records its population-phase misses in the same branch
+ * profile — MDO counters accumulate from interpretation onward and are never reset — giving a
+ * two-sided profile, no pruning, and a {@code Key2} allocated on every lookup. Treat the {@code
+ * ConcurrentHashMap} and {@code ConcurrentSkipListMap} numbers here as an upper bound that a
+ * production miss rate would erode. See {@code HashtableD2Benchmark}, where the same wrapper is
+ * not eliminated because {@code merge} keeps the present/absent decision inside the callee,
+ * leaving no caller-visible branch to prune.
+ *
  * 
        Key findings:
  *
  * 
          
- *   
        • {@code Support} and {@code ConcurrentHashtable} are the two fastest and stay close to each
- *       other on both {@code get} and {@code getOrCreate}; both avoid the {@link Key2} wrapper
- *       allocation that {@code ConcurrentHashMap} requires on every lookup.
- *   
        • {@code ConcurrentHashMap} is ~30-35% slower than {@code ConcurrentHashtable} — the {@link
- *       Key2} allocation plus two-level hash lookup adds up.
- *   
        • {@code Support} shows slightly higher throughput than {@code D2} because its primitive
- *       {@code int} K2 field avoids boxing inside the entry match on the write-path re-check.
- *   
        • {@code ConcurrentSkipListMap} is ~6× slower than {@code ConcurrentHashMap} due to tree
- *       traversal.
- *   
        • Synchronized {@code HashMap} is over 150× slower than the fastest options (8.8 vs 1573.5
+ *   
        • {@code Support} and {@code ConcurrentHashtable} are the two fastest on {@code get} (2730.0
+ *       and 2653.0 ops/us), ~60% ahead of {@code ConcurrentHashMap} (1665.0). Since the {@code
+ *       Key2} allocation is eliminated in all three (see above), that lead is the two-level hash
+ *       lookup rather than allocation.
+ *   
        • On {@code getOrCreate} the ordering inverts: {@code ConcurrentHashMap} (1647.1) overtakes
+ *       {@code ConcurrentHashtable} (1397.0), whose own {@code getOrCreate} is roughly half its
+ *       {@code get}. {@code Support} holds up (2597.4). Not root-caused here — the write-path
+ *       re-check is the obvious suspect, but it is not measured.
+ *   
        • {@code Support} edges {@code D2} on both paths, consistent with its primitive {@code int}
+ *       K2 field avoiding boxing inside the entry match on the write-path re-check.
+ *   
        • {@code ConcurrentSkipListMap} is ~9× slower than {@code ConcurrentHashMap} due to tree
+ *       traversal, though its error bar is wide (±40.1 on a 187.3 mean).
+ *   
        • Synchronized {@code HashMap} is roughly 290× slower than the fastest options (9.3 vs 2730.0
  *       ops/us) — lock contention across eight threads on a single monitor, the same magnitude seen
  *       in {@link ThreadSafeMapD1Benchmark}. Type-profile pollution is not a factor: {@code Key2}
  *       is a final class built at the call site, so C2 has an exact type and devirtualizes without

From 492066be0f0a637b8b56644b2a086046c731ea5d Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Fri, 25 Sep 2026 07:46:51 -0400
Subject: [PATCH 33/52] Document the branch-profile / escape-analysis
 interaction

A one-sided branch profile lets C2 prune the untaken branch as unstable_if, and
when that branch held the only store of an object, escape analysis then
scalar-replaces an allocation production would keep. Measured here as
get-then-guarded-computeIfAbsent, where pre-installing every key in @Setup makes
a composite key free; a real cache records population-phase misses in the same
profile and allocates on every lookup.

Also notes what this utility does not cover: branch profiles, and the ordering
dependency from JMH running arms sequentially. Flags warmUpArms as the planned
follow-on rather than adding unadopted API here.

Co-Authored-By: Claude Opus 5 (1M context) 
---
 .../datadog/trace/util/BenchmarkUtils.java    | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index 50f786d5a44..9bf747535eb 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -23,6 +23,26 @@
  * devirtualized, artificially monomorphic fast path that never occurs in the real system. The
  * pollution helpers here ({@link #warmUpHashDispatch}) exist to guard against that second problem;
  * they're not redundant with {@code Blackhole}, they cover the axis it doesn't.
+ *
+ * 
          Receiver profiles are not the only kind, and branch profiles interact with escape analysis in
+ * a way that can silently flatter a benchmark. When a branch is never taken during profiling, C2
+ * prunes it as an {@code unstable_if} uncommon trap; if that pruned branch held the only store of
+ * an object, the object becomes provably non-escaping and escape analysis scalar-replaces an
+ * allocation that production would keep. The case measured in this module is {@code map.get(key)}
+ * followed by a guarded {@code computeIfAbsent(key, ...)}: with every key pre-installed by
+ * {@code @Setup} the absent branch never runs, so a composite key costs nothing at all. A real
+ * cache records its population-phase misses in that same branch profile -- MDO counters accumulate
+ * from interpretation onward and are never reset -- so the profile is two-sided, nothing is pruned,
+ * and the key is allocated on every lookup. See {@code ThreadSafeMapD2Benchmark} for the
+ * measurement, and {@code HashtableD2Benchmark} for the contrasting shape, where {@code merge}
+ * keeps the present/absent decision inside the callee and leaves no caller-visible branch to prune.
+ *
+ * 
          Nothing here addresses branch profiles yet. The more complete approach is to exercise every
+ * benchmark arm during setup, across each of its outcomes; that also removes an ordering
+ * dependency, since JMH runs arms sequentially and whichever runs first currently shapes the shared
+ * profiles the rest inherit. A {@code warmUpArms} helper along those lines is planned as a
+ * follow-on, because adopting it changes benchmark setup and requires re-measuring whatever adopts
+ * it.
  */
 public final class BenchmarkUtils {
   private BenchmarkUtils() {}

From 1ce8fec1ad69fabef1efda360f7964df54ab7064 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Fri, 25 Sep 2026 07:51:33 -0400
Subject: [PATCH 34/52] Reframe BenchmarkUtils around fork isolation as the
 root cause

Corrects a claim added in the previous commit. I had written that arms run
sequentially and the first shapes profiles the rest inherit; that is wrong. JMH
forks a fresh JVM per benchmark method -- verified in our own run log, 29
benchmark methods and 29 VM invokers -- so no profile carries between arms.

The real problem is isolation, not ordering, and it is the root cause of all
three issues this class deals with. A fresh JVM per arm means C2 sees only that
arm's key types, only that arm's branch outcomes, and only the classes that arm
loads. So receiver profiles collapse, branch profiles go one-sided, and CHA gets
an artificially thin hierarchy. JMH forks to eliminate cross-benchmark profile
pollution, but profile pollution is the production condition.

Restructured the class javadoc to lead with that and enumerate the three
consequences, demoting Blackhole to the orthogonal note it is. Also records that
a future warmUpArms should shuffle arm order with a fixed seed, since
compilation can trigger part-way through a fixed round-robin and a regular
pattern is unrealistically easy on the hardware branch predictor.

Co-Authored-By: Claude Opus 5 (1M context) 
---
 .../datadog/trace/util/BenchmarkUtils.java    | 74 ++++++++++++-------
 1 file changed, 49 insertions(+), 25 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index 9bf747535eb..72758640d4d 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -16,33 +16,57 @@
 /**
  * Shared setup helpers for JMH benchmarks in this module.
  *
- * 
          {@link Blackhole} solves one correctness problem: it stops the JIT from proving a result is
- * dead and eliminating the code that produced it. It says nothing about a second, separate problem
- * -- whether a call site's receiver-type profile still looks like production once measurement
- * starts. A benchmark can consume every result through a {@code Blackhole} and still measure a
- * devirtualized, artificially monomorphic fast path that never occurs in the real system. The
- * pollution helpers here ({@link #warmUpHashDispatch}) exist to guard against that second problem;
- * they're not redundant with {@code Blackhole}, they cover the axis it doesn't.
+ * 
          These exist to compensate for forking. JMH runs each benchmark method in a fresh JVM, which is
+ * right for measurement -- one arm cannot contaminate another's numbers -- but it means every arm
+ * is compiled inside a process that has only ever executed that one code path, with that one arm's
+ * key types and that one arm's branch outcomes. Production is the reverse: a single JVM runs
+ * everything. So the harness hands C2 a maximally specialized view of the world, and the
+ * compilation that results can be better than anything achievable in a real application. Put
+ * another way, JMH forks to eliminate cross-benchmark profile pollution, but profile pollution
+ * is the production condition.
  *
- * 
          Receiver profiles are not the only kind, and branch profiles interact with escape analysis in
- * a way that can silently flatter a benchmark. When a branch is never taken during profiling, C2
- * prunes it as an {@code unstable_if} uncommon trap; if that pruned branch held the only store of
- * an object, the object becomes provably non-escaping and escape analysis scalar-replaces an
- * allocation that production would keep. The case measured in this module is {@code map.get(key)}
- * followed by a guarded {@code computeIfAbsent(key, ...)}: with every key pre-installed by
- * {@code @Setup} the absent branch never runs, so a composite key costs nothing at all. A real
- * cache records its population-phase misses in that same branch profile -- MDO counters accumulate
- * from interpretation onward and are never reset -- so the profile is two-sided, nothing is pruned,
- * and the key is allocated on every lookup. See {@code ThreadSafeMapD2Benchmark} for the
- * measurement, and {@code HashtableD2Benchmark} for the contrasting shape, where {@code merge}
- * keeps the present/absent decision inside the callee and leaves no caller-visible branch to prune.
+ * 
          That surfaces three ways. This class addresses the first two partially and the third not at
+ * all:
  *
- * 
          Nothing here addresses branch profiles yet. The more complete approach is to exercise every
- * benchmark arm during setup, across each of its outcomes; that also removes an ordering
- * dependency, since JMH runs arms sequentially and whichever runs first currently shapes the shared
- * profiles the rest inherit. A {@code warmUpArms} helper along those lines is planned as a
- * follow-on, because adopting it changes benchmark setup and requires re-measuring whatever adopts
- * it.
+ * 
            
+ *   
          • Receiver-type profiles collapse toward a single key type, so shared JDK dispatch
+ *       sites look monomorphic. {@link #warmUpHashDispatch} drives several key classes through
+ *       them.
+ *   
          • Class-hierarchy analysis sees only the implementations one arm happens to load, so
+ *       C2 can devirtualize calls a real application leaves polymorphic. Loading the decoy
+ *       collections widens the hierarchy.
+ *   
          • Branch profiles go one-sided, because only one arm's outcomes ever occur. Nothing
+ *       here addresses that.
+ * 
          
+ *
+ * 
          {@link Blackhole} is orthogonal to all of this. It stops the JIT proving a result is dead, and
+ * says nothing about whether the surrounding code was compiled realistically -- a benchmark can
+ * consume every result through a {@code Blackhole} and still measure a devirtualized fast path that
+ * cannot occur in production.
+ *
+ * 
          One-sided branch profiles also interact with escape analysis in a way that can silently
+ * flatter a benchmark. When a branch is never taken during profiling, C2 prunes it as an {@code
+ * unstable_if} uncommon trap; if that pruned branch held the only store of an object, the object
+ * becomes provably non-escaping and escape analysis scalar-replaces an allocation that production
+ * would keep. The case measured in this module is {@code map.get(key)} followed by a guarded {@code
+ * computeIfAbsent(key, ...)}: with every key pre-installed by {@code @Setup} the absent branch
+ * never runs, so a composite key costs nothing at all. A real cache records its population-phase
+ * misses in that same branch profile -- MDO counters accumulate from interpretation onward and are
+ * never reset -- so the profile is two-sided, nothing is pruned, and the key is allocated on every
+ * lookup. See {@code ThreadSafeMapD2Benchmark} for the measurement, and {@code
+ * HashtableD2Benchmark} for the contrasting shape, where {@code merge} keeps the present/absent
+ * decision inside the callee and leaves no caller-visible branch to prune.
+ *
+ * 
          The more complete approach is to exercise every benchmark arm during setup, across each of its
+ * outcomes, so that each fork's profiles reflect the whole class rather than the one arm it is
+ * about to measure. That is what restores the mix the fork removed.
+ *
+ * 
          Such a helper should drive the arms in a shuffled order rather than a fixed round-robin, since
+ * compilation can trigger part-way through a warmup and would otherwise see whichever arm dominates
+ * that point in the sequence; a regular pattern also gives the hardware branch predictor an
+ * unrealistically easy time. Seeding the shuffle keeps it irregular but reproducible. A {@code
+ * warmUpArms} along those lines is planned as a follow-on, because adopting it changes benchmark
+ * setup and requires re-measuring whatever adopts it.
  */
 public final class BenchmarkUtils {
   private BenchmarkUtils() {}

From 59c58ebf8207fc931dc952f631e1f395f6858ef4 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Fri, 25 Sep 2026 15:05:32 -0400
Subject: [PATCH 35/52] Fix perf-review J12 check and clean up
 pollution-migration leftovers

From /techdebt and /perf-review over the branch.

The J12 benchmark-validity check added in this branch had two problems. It
instructed reviewers to call BenchmarkUtils.polluteHashDispatch(), which this
same branch made private -- the public entry point is warmUpHashDispatch(
Blackhole). And it cited "synchronized HashMap collapsed ~70% once its dispatch
was made megamorphic" as its evidence, which is the attribution this PR
retracted: those keys are String, a final class, so C2 devirtualizes on an exact
type and never consults the profile. The gap there is lock contention.

Rewrote J12 around the actual discriminator -- whether the receiver type is
statically deducible at the benchmark's own call site -- and noted explicitly
that one key class per benchmark is normal, since a production Map
call site is monotyped too. Evidence is now FlatHashtableIteratorBenchmark's
four-implementor strategy dispatch (37.9 to 18.1 M ops/s when poisoned) against
the String-keyed maps that were unaffected. Downgraded to flag-as-measure, and
corrected the fix to Level.Trial only, since the helper runs 50,000
allocation-heavy passes.

Also removed two orphan comments in ImmutableSet/ImmutableMapBenchmark that
described a single pollution call deleted earlier in the branch, and widened the
boxed-constructor suppression to include "removal", which is the actual warning
category for @Deprecated(forRemoval = true).

Co-Authored-By: Claude Opus 5 (1M context) 
---
 .../skills/perf-review/references/checks.md   |  5 ++--
 .../skills/perf-review/references/guide.md    | 27 +++++++++++++------
 .../datadog/trace/util/BenchmarkUtils.java    |  6 +++--
 .../trace/util/ImmutableMapBenchmark.java     |  3 ---
 .../trace/util/ImmutableSetBenchmark.java     |  3 ---
 5 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/.agents/skills/perf-review/references/checks.md b/.agents/skills/perf-review/references/checks.md
index 229ce363f48..d5082d62215 100644
--- a/.agents/skills/perf-review/references/checks.md
+++ b/.agents/skills/perf-review/references/checks.md
@@ -51,9 +51,8 @@ Refines the universal checks with JVM mechanics. Quarantined here, for the Java
 - **J9 — `Objects.hash(...)` varargs / boxing hash on a hot path → `HashingUtils`** *(refines #1)*. The allocation is specific to the **varargs/boxing forms**: `Objects.hash(a, b, …)` allocates an `Object[]` per call and **boxes every primitive** arg; same for boxing primitives into a `new Object[]{…}` (or `Arrays.hashCode` over such an array). Per-span tag/key building, or a hot value object's `hashCode()` built this way, → a guaranteed per-call allocation + boxing. Fix: `datadog.trace.util.HashingUtils` — primitive `hash(long/int/boolean/char/…)` overloads (no boxing), `hash(Object,Object)` and `hash(int,int)` combiners (no array); for >2 fields fold pairwise through `hash(int,int)` (there is no varargs form, by design). flag-with-confidence for the varargs/boxing form — SEV-2/3. **Do NOT flag allocation-free combines** — a hand-rolled `31*h + Long.hashCode(x)` / `31*h + intField`, or `Arrays.hashCode` over an *existing primitive array*, allocates nothing (`HashingUtils` is itself 31-based); flagging them would recommend replacing already-correct code.
 - **J10 — hot-path `String.format` / string munging → `Strings` (+ `SubSequence`)** *(refines #2)*. `String.format` parses the format string, boxes its args, and allocates on every call — never on a hot path; hand-rolled case-conversion, class/resource-name munging, blank-checks, and truncation recomputed per call qualify too. Fix: `datadog.trace.util.Strings` — allocation-aware `replace`/`truncate(CharSequence)`/`isBlank`/`getResourceName`/`getClassName`/…; for **transient substring compares** prefer a `SubSequence` view (J7); for plain assembly, direct concatenation beats `format`. flag-with-confidence for `String.format` on a hot path; flag-as-measure for borderline munging — SEV-2/3.
 - **J11 — composite / multi-dimensional key maps on a hot path → `Hashtable` / `ConcurrentHashtable`** *(refines #1, #3)*. `Map>` nesting, or a `HashMap` keyed by a composite key (client-side stats, per-`(service, operation, …)` aggregation), allocates nested maps + `Entry` objects + boxes keys on the hot aggregation path. Fix: `datadog.trace.util.Hashtable` (single-threaded, composite-key D1/D2 tables — landed) or `datadog.trace.util.ConcurrentHashtable` (lock-free concurrent, **coming**) — positional composite keys, fewer allocations. flag-as-measure — SEV-2/3.
-- **J12 — Benchmark validity: pollute Map/Set dispatch before trusting a JMH comparison** *(methodology check on a submitted benchmark, not a production hot-path finding — refines the flag-as-measure posture in J2)*. A JMH benchmark comparing `HashMap`/`HashSet`/`ConcurrentHashMap` (or `TreeMap`/`TreeSet`/`ConcurrentSkipListMap`, or the JDK's immutable `Map.of`/`Set.of` `MapN`/`SetN`) against another collection, that only ever looks up one key class for the whole run, leaves that collection's internal `hashCode()`/`equals()` (or `compareTo`) dispatch site artificially monomorphic — a state production rarely reaches, since those call sites are shared JVM-wide across every key type the whole process uses. An unpolluted benchmark can inflate the affected structure's throughput and flip or exaggerate a comparison (this is exactly what PR #12298 found and fixed: synchronized `HashMap` collapsed ~70% once its dispatch was made megamorphic). Before trusting such a benchmark's numbers, check its `@Setup` calls `datadog.trace.util.BenchmarkUtils.polluteHashDispatch()` (covers `HashSet`/`HashMap`/`ConcurrentHashMap`/`SetN`/`MapN` `equals()` dispatch and `TreeMap`/`TreeSet`/`ConcurrentSkipListMap` `compareTo` dispatch) before the measured operations run. **flag-with-confidence** when a new or edited Map/Set JMH benchmark in this module omits it — fix: call `BenchmarkUtils.polluteHashDispatch()` in `@Setup(Level.Trial)`/`Level.Iteration`, ahead of the benchmarked calls.
-
-- **Toolkit availability — cite only what exists.** Available today: `Strings`, `SubSequence`, `HashingUtils`, `Hashtable`, `BenchmarkUtils.polluteHashDispatch()`/`polluteCompareToDispatch()` for JMH setup (all `datadog.trace.util`), `RE2J` (`com.google.re2j`). Coming (name as "coming", don't imply it's present): `ConcurrentHashtable`, `StringIndex` (immutable string set/map), `UTF8BytesString.Cache` (recurring-string interner), wider `IntegerCache` (http-status/port boxing), `DDCache` inlining. **J7–J11 route an *existing* #1/#2/#3 finding to a reusable fix — they are not new flag-triggers. Don't raise a finding you wouldn't have raised anyway; the posture (precision, silent-when-unsure, findings-cap-scales-with-diff — see `SKILL.md`) is unchanged.**
+- **J12 — Benchmark validity: is the dispatch under test actually profile-dependent?** *(methodology check on a submitted benchmark, not a production hot-path finding — refines the flag-as-measure posture in J2)*. JMH forks a fresh JVM per benchmark method, so each arm is compiled in a process that has only ever executed that one arm — leaving shared `hashCode()`/`equals()`/`compareTo` call sites unrealistically clean compared with production, where erasure makes them serve every map in the application. This only affects the measurement when the receiver type is **not statically deducible at the benchmark's own call site**: if the key is a final class (`String`) or an exact type from an allocation, C2 devirtualizes on that proof and never consults the profile, so pollution is a no-op that only adds setup allocation. Note that one key class per benchmark is normal and correct — a production `Map` call site is monotyped too; variety at the caller is not what makes the internal site megamorphic. **flag-as-measure** when a Map/Set JMH benchmark's measured operation dispatches on a non-final key type, or reaches a strategy/comparator interface with several loaded implementors, and its `@Setup` does not pollute — fix: call `BenchmarkUtils.warmUpHashDispatch(Blackhole)` from `@Setup(Level.Trial)` (trial level, not per-iteration: it runs 50,000 allocation-heavy passes). PR #12298 measured both sides — `FlatHashtableIteratorBenchmark`'s four-implementor strategy dispatch fell 37.9 → 18.1 M ops/s when its profile was poisoned, while the `String`-keyed map benchmarks in the same module were unaffected, with LogCompilation confirming the dispatch resolved statically.
+- **Toolkit availability — cite only what exists.** Available today: `Strings`, `SubSequence`, `HashingUtils`, `Hashtable`, `BenchmarkUtils.warmUpHashDispatch(Blackhole)` for JMH setup (all `datadog.trace.util`), `RE2J` (`com.google.re2j`). Coming (name as "coming", don't imply it's present): `ConcurrentHashtable`, `StringIndex` (immutable string set/map), `UTF8BytesString.Cache` (recurring-string interner), wider `IntegerCache` (http-status/port boxing), `DDCache` inlining. **J7–J11 route an *existing* #1/#2/#3 finding to a reusable fix — they are not new flag-triggers. Don't raise a finding you wouldn't have raised anyway; the posture (precision, silent-when-unsure, findings-cap-scales-with-diff — see `SKILL.md`) is unchanged.**
 - **J5 — Cardinality-sensitive aggregator** *(domain-specialized #3)*. Some structures are invisible to the generic "unbounded collection" check because the risk is *cardinality*, not raw size: a config- or user-driven value (tag key, resource name, HTTP URL) feeding a **cardinality-sensitive aggregator** (e.g. the conflating metrics aggregator — each unique label combination = one aggregate; a `maxAggregates` cap bounds OOM but high-cardinality input *thrashes* it: constant eviction, garbled metrics). flag-with-confidence when config/user-driven values reach an aggregator with a per-key budget — **SEV-1** (same class as unbounded memory: correctness + heap impact). Fix: bound the source cardinality before it enters the aggregator, or use sentinel substitution for over-cap values. This surfaced on merged production code more than once in back-test calibration — the capstone pattern where the bot's value concentrates.
 
 ## Instrumentation (ByteBuddy Advice) idioms — dd-trace-java-specific fixes
diff --git a/.agents/skills/perf-review/references/guide.md b/.agents/skills/perf-review/references/guide.md
index 2f77a78b22a..d1532495305 100644
--- a/.agents/skills/perf-review/references/guide.md
+++ b/.agents/skills/perf-review/references/guide.md
@@ -161,14 +161,25 @@ escape analysis eliminates *local* short-lived allocations, but only when the ob
 Stored in a map, returned, captured by a lambda, or passed to a non-inlined virtual call: it
 escapes, and it's real.
 
-**Map/Set JMH benchmarks that never pollute dispatch — treat as unverified.** A benchmark that
-looks up only one key class for its whole run leaves the collection's internal
-`hashCode()`/`equals()` (or `compareTo` for sorted maps/sets) call site artificially
-monomorphic — production hits that same shared call site with whatever key types the whole
-process uses, so it's realistically almost always megamorphic. An unpolluted benchmark can
-overstate a structure's throughput and flip a comparison (PR #12298: a synchronized `HashMap`
-collapsed ~70% once pollution was added). Check that the benchmark's `@Setup` calls
-`datadog.trace.util.BenchmarkUtils.polluteHashDispatch()` before trusting its numbers.
+**Map/Set JMH benchmarks whose dispatch is actually reachable — treat as unverified.** JMH forks a
+fresh JVM per benchmark method, so each arm is compiled in a process that has only ever seen that
+one arm's key types. A collection's internal `hashCode()`/`equals()` (or `compareTo`) call site is
+shared JVM-wide in production and hit with every key type the process uses, so the benchmark's
+version of it is unrealistically clean.
+
+But this only matters when the receiver type is **not statically deducible at the benchmark's own
+call site**. If the key is a final class (`String`) or an exact type from an allocation, C2
+devirtualizes on that proof and never consults the profile — pollution is then a no-op, and
+demanding it only adds setup allocation. Having one key class per benchmark is normal and correct;
+a real `Map` call site is monotyped too. Erasure is what makes the *internal* site
+megamorphic, not variety at the caller.
+
+PR #12298 measured both sides: `FlatHashtableIteratorBenchmark`'s strategy dispatch fell 37.9 → 18.1
+M ops/s (2.1x) when its profile was poisoned, because the strategy is an interface with four loaded
+implementors; the `String`-keyed map benchmarks in the same module were unaffected, and
+LogCompilation confirmed the dispatch was resolved statically. Check that `@Setup` calls
+`datadog.trace.util.BenchmarkUtils.warmUpHashDispatch(Blackhole)` when the dispatch is genuinely
+profile-dependent.
 
 **EA claims for scope/wrapper objects spanning I/O — treat as unverified.** A microbenchmark
 tight-loop can show zero allocation for a scope or wrapper object because C2 inlines through
diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index 72758640d4d..57a957f33a4 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -248,8 +248,10 @@ private static void polluteComparatorDispatch(Blackhole bh) {
    * @return a distinct-but-equal copy, or {@code key} itself if it has no distinct-but-equal form
    */
   @CompilerControl(CompilerControl.Mode.DONT_INLINE)
-  @SuppressWarnings(
-      "deprecation") // boxed-type constructors: only way to force a non-cached instance
+  @SuppressWarnings({
+    "deprecation",
+    "removal"
+  }) // boxed-type constructors: only way to force a non-cached instance
   private static Object distinctEqualCopy(Object key) {
     if (key instanceof String) {
       return new String((String) key);
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
index 58edce8873c..bbebbe4805e 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
@@ -144,9 +144,6 @@ static void fill(Map map) {
 
   @Setup(Level.Trial)
   public void setUp() {
-    // Superseded by Cursor#warmUpPollution's heavier front-load below -- this single call isn't
-    // enough on its own to drive HotSpot's tiered compiler through both C1 and C2 on the shared
-    // hash-dispatch call sites (see BenchmarkUtils#warmUpHashDispatch).
     hashMap = new HashMap<>();
     fill(hashMap);
     linkedHashMap = new LinkedHashMap<>();
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java
index fcc2f5209cd..e6ef49b7eff 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableSetBenchmark.java
@@ -140,9 +140,6 @@ static String[] newMisses() {
 
   @Setup(Level.Trial)
   public void setUp() {
-    // Superseded by Cursor#warmUpPollution's heavier front-load below -- this single call isn't
-    // enough on its own to drive HotSpot's tiered compiler through both C1 and C2 on the shared
-    // hash-dispatch call sites (see BenchmarkUtils#warmUpHashDispatch).
     array = STRINGS;
     sortedArray = Arrays.copyOf(STRINGS, STRINGS.length);
     Arrays.sort(sortedArray);

From 754a5ed0cdcfb03ca7169532d7036742825525c9 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Fri, 25 Sep 2026 15:50:54 -0400
Subject: [PATCH 36/52] Run hash-dispatch pollution once per JVM instead of
 once per thread

Nine of the eleven benchmarks wire warmUpHashDispatch into a @State(Scope.Thread)
setup, which JMH runs once per thread -- so at @Threads(8) the pollution work was
being done eight times. Receiver profiles live in the per-method MDO and are
JVM-global, so one pass is sufficient however many threads ask for it; the other
seven were pure waste, and the allocation they generate lands before measurement
starts.

Guarded with a CAS rather than restructuring the call sites, so it holds
regardless of which state scope a caller uses and stays correct if someone later
wires it into Level.Iteration.

Note this changes setup behaviour for the nine Scope.Thread benchmarks, so their
tables were produced under the old 8x pollution. TagMapAccessBenchmark and
ThreadSafeMapCounterBenchmark were already Scope.Benchmark and are unaffected --
which also rules out 8x pollution as the explanation for TagMapAccess's
insert_via_ledger drop.

Co-Authored-By: Claude Opus 5 (1M context) 
---
 .../java/datadog/trace/util/BenchmarkUtils.java    | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index 57a957f33a4..5479d1b9cf0 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -10,6 +10,7 @@
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.openjdk.jmh.annotations.CompilerControl;
 import org.openjdk.jmh.infra.Blackhole;
 
@@ -90,6 +91,16 @@ private BenchmarkUtils() {}
    */
   private static final int WARM_UP_ITERATIONS = 50_000;
 
+  /**
+   * Guards {@link #warmUpHashDispatch} so the passes run once per JVM rather than once per caller.
+   * Receiver profiles live in the per-method MDO and are JVM-global, so a single pass is sufficient
+   * no matter how many threads or state classes ask for it. Most benchmarks here wire the call into
+   * a {@code @State(Scope.Thread)} setup, which JMH runs once per thread -- without this guard,
+   * {@code @Threads(8)} would perform eight times the pollution work, and the allocation it
+   * generates before measurement is not free.
+   */
+  private static final AtomicBoolean WARMED_UP = new AtomicBoolean();
+
   /**
    * Exercises shared collection methods with several key classes before benchmark warmup.
    *
@@ -118,6 +129,9 @@ private BenchmarkUtils() {}
    * stops the JIT from tracing a decoy key's type back to its origin through static inference.
    */
   public static void warmUpHashDispatch(Blackhole bh) {
+    if (!WARMED_UP.compareAndSet(false, true)) {
+      return;
+    }
     for (int i = 0; i < WARM_UP_ITERATIONS; ++i) {
       polluteHashDispatch(bh);
       polluteCompareToDispatch(bh);

From d6c6284ef4de2323b35004eac885773ca0a47e24 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Sun, 27 Sep 2026 20:29:16 -0400
Subject: [PATCH 37/52] Refresh tables from the once-per-JVM run and rule out
 setup-allocation churn

Pollution now runs once per JVM rather than once per thread, so the nine
Scope.Thread benchmarks needed re-measuring. Zulu 17, -prof gc, same machine and
fork config as the previous run.

The interesting result is a negative one. Three files had been carrying
"warmUpHashDispatch allocates heavily before measurement, which can shift GC
state for the whole trial" as an explicitly unmeasured candidate for otherwise
unexplained drops. The guard produced an 8x reduction in that allocation with
everything else held constant, and nothing moved: update_hashMap 686.0 to 689.6,
update_hashtable 2770.7 to 2796.3, iterate_hashMap 19.83 to 19.84,
get_concurrentHashMap 1830.7 to 1837.1, get_support 2730.0 to 2731.5. All inside
their intervals. That candidate is now tested and ruled out rather than left
hanging, and HashtableD1 carries the comparison with D2 pointing at it.

HashtableD2's update margin lands at ~5.5x and its add arms remain a tie, now
with the hashtable ahead on the means but overlapping intervals. One arm moved
outside its interval -- D2 update_hashtable, 2492.7 to 2346.8 -- on the
zero-allocation path, so a GC explanation runs the wrong way; recorded as
run-to-run drift.

Refreshed throughput only for the four classes whose config is comparable
between the two runs. For CaseInsensitiveMap, ImmutableMap and SingleThreadedMap
the existing tables were taken at a different fork count, so only the allocation
figures are added -- bytes/op is deterministic and config-insensitive:

- CaseInsensitiveMap: lookup_hashMap is the only allocating lookup arm at
  25.5 B/op for the folded String, against zero elsewhere. The variance is real
  signal, since toLowerCase() returns this for already-lower-case input.
- ImmutableMap: the forEach-beats-iterator finding now has a mechanism --
  iterate_tagMap 40 B/op versus zero for iterate_tagMap_forEach.
- SingleThreadedMap: clone_tagMap 208 B/op against clone_hashMap 488.

Co-Authored-By: Claude Opus 5 (1M context) 
---
 .../util/CaseInsensitiveMapBenchmark.java     |  8 ++++
 .../trace/util/HashtableD1Benchmark.java      | 30 ++++++++-----
 .../trace/util/HashtableD2Benchmark.java      | 38 ++++++++--------
 .../trace/util/ImmutableMapBenchmark.java     |  8 +++-
 .../util/SingleThreadedMapBenchmark.java      |  3 +-
 .../trace/util/ThreadSafeMapD1Benchmark.java  | 24 +++++-----
 .../trace/util/ThreadSafeMapD2Benchmark.java  | 44 ++++++++++---------
 7 files changed, 90 insertions(+), 65 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
index 39c6bb169f8..4d0ef8b479b 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
@@ -50,6 +50,14 @@
  * decisive) — tight enough that {@code hashMap}'s lead over {@code flatHashtable} (488.5 vs 380.3,
  * ~28%) is a real, if not perfectly clean-cut, result rather than noise.
  *
+ * 
          Measured with {@code -prof gc}: {@code lookup_hashMap} is the only lookup arm that
+ * allocates, at 25.5 ± 4.6 B/op for the folded {@code String}, against ≈0 for {@code
+ * flatHashtable}, {@code flatHashtable_lowLoad}, {@code treeMap} and the baseline. That quantifies
+ * the allocation win claimed above. The variance is itself informative: {@code toLowerCase()}
+ * returns {@code this} when a string is already lower-case, so the figure tracks the mixed-case
+ * fraction of the key set rather than a fixed per-lookup cost. Allocation came from a later run
+ * than the throughput table, so the two are not a matched pair.
+ *
  * 
          Takeaway, revised. {@code HashMap} keyed on {@code toLowerCase()} is faster than {@code
  * FlatHashtable} for this lookup shape, not merely comparable to it as earlier (noisier) runs
  * suggested. {@code FlatHashtable} still wins on allocation — it is the zero-allocation option, and
diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
index 645d717c79e..28112a8071c 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
@@ -43,7 +43,7 @@
  * entry; the HashMap path boxes a {@code Long} on every {@code merge}. Measured with {@code -prof
  * gc} on Zulu 17, {@code update_hashMap} allocates 24.000 ± 0.001 B/op — exactly one boxed {@code
  * Long} (12-byte header plus an 8-byte value, aligned to 24) — against ≈0 B/op for {@code
- * update_hashtable}, and 852 collections over the run against none. The GC pressure is measured
+ * update_hashtable}, and 801 collections over the run against none. The GC pressure is measured
  * rather than inferred from throughput. This is the headline case for {@code Hashtable}: a simple
  * counter/tally with a primitive value is exactly where HashMap's autoboxing tax bites hardest, and
  * {@code Hashtable.D1} sidesteps it entirely by mutating a field on the retrieved entry in place.
@@ -75,10 +75,16 @@
  * sharpens the {@code Object}-declared key to an exact type and devirtualizes {@code
  * hashCode()}/{@code equals()} without consulting the polluted profile. Pollution therefore cannot
  * explain a drop on either side. The JDK and machine were held constant, so what remains is
- * uncontrolled run-to-run variation plus one concrete candidate: {@code warmUpHashDispatch} itself
- * allocates heavily before measurement starts, which can shift GC state for the whole trial.
- * Neither was measured. The relative conclusion (D1 dominates {@code update}, is roughly
- * comparable on {@code add}, ties on {@code iterate}) is unchanged either way.
+ * uncontrolled run-to-run variation.
+ *
+ * 
          The other candidate — that {@code warmUpHashDispatch}'s own heavy pre-measurement allocation
+ * shifts GC state for the trial — has since been tested and ruled out. Pollution used to run
+ * once per thread, so at {@code @Threads(8)} it did eight times the work; it now runs once per JVM.
+ * Re-measuring across that 8x reduction moved nothing: {@code update_hashMap} 686.0 to 689.6,
+ * {@code update_hashtable} 2770.7 to 2796.3, {@code iterate_hashMap} 19.83 to 19.84, all well
+ * inside their intervals. Setup allocation is therefore not what moved these numbers. The
+ * relative conclusion (D1 dominates {@code update}, is roughly comparable on {@code add},
+ * ties on {@code iterate}) is unchanged throughout.
  *
  * 
          Separately rerun on Zulu 17.0.7 (native AArch64, same machine, pollution wiring unchanged; JMH
  * auto-detected the cheap "compiler" Blackhole mode here, unlike JDK 8, so absolute numbers below
@@ -87,12 +93,12 @@
  *
  * 
          {@code
  * Benchmark            ops/us            B/op   gc.count
- * add_hashMap        1517.5 ± 242.9      32.0       1820
- * add_hashtable      1302.1 ± 403.9      40.0       1933
- * update_hashMap      686.0 ± 140.4      24.0        852
- * update_hashtable   2770.7 ± 169.4       ~0          ~0
- * iterate_hashMap      19.8 ±   0.6      40.0         55
- * iterate_hashtable    79.6 ±   9.6       ~0          ~0
+ * add_hashMap        1658.6 ± 190.5      32.0       1839
+ * add_hashtable      1358.0 ± 213.6      40.0       1710
+ * update_hashMap      689.6 ± 140.8      24.0        801
+ * update_hashtable   2796.3 ±  79.7       ~0          ~0
+ * iterate_hashMap      19.8 ±   0.3      40.0         76
+ * iterate_hashtable    81.5 ±   2.5       ~0          ~0
  * }
          
  *
  * 
          Allocation is measured with {@code -prof gc} and decomposes exactly: 24 B/op for {@code
@@ -109,7 +115,7 @@
  * iterate_hashtable} also now clearly wins (~4.0x), flipping from JDK 8's "wash" — HashMap's {@code
  * entrySet()} iterator does more per-entry work than a modern JIT's allocation improvements erase.
  * {@code add} is the one case that flips the other way: {@code add_hashMap} leads on the means
- * (1517.5 vs 1302.1), though both error bars are wide enough to overlap, so treat that one as
+ * (1658.6 vs 1358.0), though both error bars are wide enough to overlap, so treat that one as
  * undecided rather than a HashMap win. Net takeaway: {@code Hashtable} is a strong substitute for
  * {@code HashMap} particularly for simple counter/tally use cases with a primitive value, where
  * avoiding the per-update boxing allocation pays off even on a JVM with much better allocation
diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
index a61d0c86a2e..50f8d07c2de 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
@@ -47,7 +47,7 @@
  * Measured with {@code -prof gc} on Zulu 17, {@code update_hashMap} allocates 48.000 ± 0.001 B/op,
  * which decomposes exactly: 24 for the boxed {@code Long} and 24 for the {@code Key2} wrapper
  * (12-byte header, two references, one {@code int}). {@code update_hashtable} allocates ≈0 B/op.
- * Collections over the run were 420 against none.
+ * Collections over the run were 890 against none.
  *
  * 
          That 48 also answers a common assumption: escape analysis does not eliminate the
  * temporary {@code Key2}, even though every key in this benchmark is already present and the
@@ -88,11 +88,12 @@
  * is built at the call site with {@code new Key2(...)}, so C2 has an exact type either way and
  * devirtualizes {@code hashCode()}/{@code equals()} without consulting the polluted profile.
  * Pollution therefore cannot explain a move on either side. With the JDK and machine held constant,
- * what remains is uncontrolled run-to-run variation plus one concrete candidate: {@code
- * warmUpHashDispatch} itself allocates heavily before measurement starts, which can shift GC state
- * for the whole trial. Neither was measured. Treat these two runs as not directly comparable on
- * absolute numbers. The relative conclusion (D2 dominates {@code update}, wins {@code add}
- * by avoiding the {@code Key2} allocation, ties on {@code iterate}) is unchanged either way.
+ * what remains is uncontrolled run-to-run variation. The other candidate — that {@code
+ * warmUpHashDispatch}'s own heavy pre-measurement allocation shifts GC state for the trial — has
+ * since been tested and ruled out; see {@link HashtableD1Benchmark} for the 8x-to-1x comparison
+ * that moved nothing. Treat these two runs as not directly comparable on absolute numbers. The
+ * relative conclusion (D2 dominates {@code update}, leads {@code add} on JDK 8 by avoiding
+ * the {@code Key2} allocation, ties on {@code iterate}) is unchanged either way.
  *
  * 
          Separately rerun on Zulu 17.0.7 (native AArch64, same machine, pollution wiring unchanged).
  * JMH auto-detected the cheap "compiler" Blackhole mode on Java 17 (its log explicitly warns that
@@ -103,12 +104,12 @@
  *
  * 
          {@code
  * Benchmark            ops/us            B/op   gc.count
- * add_hashMap        1009.6 ± 190.2      56.0       1819
- * add_hashtable      1018.9 ± 235.0      40.0       1589
- * update_hashMap      463.0 ±  95.7      48.0       1064
- * update_hashtable   2492.7 ±  51.9       ~0          ~0
- * iterate_hashMap      19.8 ±   0.3      40.0         60
- * iterate_hashtable    72.0 ±   1.6       ~0          ~0
+ * add_hashMap         987.3 ± 322.9      56.0       1797
+ * add_hashtable      1163.6 ± 143.5      40.0       1709
+ * update_hashMap      427.9 ±  64.0      48.0        890
+ * update_hashtable   2346.8 ±  31.7       ~0          ~0
+ * iterate_hashMap      19.8 ±   0.5      40.0         76
+ * iterate_hashtable    71.7 ±   2.1       ~0          ~0
  * }
          
  *
  * 
          These numbers postdate the switch from {@code Objects.hash} to {@link
@@ -122,17 +123,18 @@
  * the {@code Key2} (24) — see the escape-analysis note above. Both hashtable paths that avoid the
  * wrapper allocate nothing on {@code update} and {@code iterate}.
  *
- * 
          {@code update_hashtable} wins by ~5.4x — down from ~26x on JDK 8, and also down from the
+ * 
          {@code update_hashtable} wins by ~5.5x — down from ~26x on JDK 8, and also down from the
  * ~11.6x this file previously reported. The difference is the {@code Objects.hash} fix: removing
  * that varargs {@code Object[]} took 24 B/op off {@code update_hashMap} and roughly doubled its
- * throughput (196.7 to 463.0). {@code iterate_hashtable} wins ~3.6x, flipping JDK 8's wash —
+ * throughput (196.7 to 427.9). {@code iterate_hashtable} wins ~3.6x, flipping JDK 8's wash —
  * HashMap's {@code entrySet()} iterator does more per-entry work than a modern JIT's allocation
  * improvements erase.
  *
- * 
          {@code add} is now a tie (1018.9 vs 1009.6, comfortably inside both error bars), where this
- * file previously claimed a ~1.8x hashtable win. That claim was an artifact of the varargs
- * allocation in the old {@code Key2} constructor; with it gone the two are indistinguishable on the
- * insert path, which makes sense — both allocate one entry per insert.
+ * 
          {@code add} is a tie (1163.6 vs 987.3; the hashtable leads on the means but {@code
+ * add_hashMap}'s interval is ±322.9, so the two overlap), where this file previously claimed a
+ * ~1.8x hashtable win. That claim was an artifact of the varargs allocation in the old {@code Key2}
+ * constructor; with it gone the two are indistinguishable on the insert path, which makes sense —
+ * both allocate one entry per insert.
  *
  * 
          Net takeaway, consistent with {@link HashtableD1Benchmark}: {@code Hashtable} is a strong
  * substitute for {@code HashMap} for counter/tally use cases with a primitive value, where avoiding
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
index bbebbe4805e..0037de6abd2 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ImmutableMapBenchmark.java
@@ -80,7 +80,13 @@
  *       instances.
  *   
        • {@code stringIndex} is also slightly better than {@code hashMap} when used as an immutable
  *       map (1452.4 vs 1240.9 on {@code get}).
- *   
        • {@code TagMap.forEach} is about 46% faster than its iterator (146.69 vs. 100.66 M ops/s).
+ *   
        • {@code TagMap.forEach} is about 46% faster than its iterator (146.69 vs. 100.66 M ops/s),
+ *       and {@code -prof gc} supplies the mechanism: {@code iterate_tagMap} allocates 40 B/op while
+ *       {@code iterate_tagMap_forEach} allocates ≈0. {@code iterate_hashMap} is also 40 B/op,
+ *       whereas {@code iterate_linkedHashMap}, {@code iterate_treeMap} and {@code
+ *       iterate_tracerImmutableMap} allocate nothing. Every {@code get} arm is allocation-free,
+ *       including the distinct-key variants. Allocation was profiled in a later run than the
+ *       throughput table above, so the two are not a matched pair on absolute numbers.
  * 
        
  */
 // @Fork(5): get_tracerImmutableMap* (MapN reached via interface dispatch) is JIT-bimodal at fewer
diff --git a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java
index 51ade692ce6..bd3c0e83cc6 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/SingleThreadedMapBenchmark.java
@@ -62,7 +62,8 @@
  *
  * clone_tagMap                  301.8   clone_treeMap                  101.5
  * clone_hashMap                  61.2   clone_synchronizedHashMap       54.9
- * clone_linkedHashMap            50.8
+ * clone_linkedHashMap            50.8   (clone B/op: tagMap 208, hashMap 488,
+ *                                        linkedHashMap 536, treeMap 480)
  *
  * get_flatHashtable             1583.7  get_hashMap                    1352.1
  * get_synchronizedHashMap        843.2
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
index d4a1d85d1bb..9a1d440531d 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
@@ -42,14 +42,14 @@
  *
  * 
        {@code
  * Benchmark                              ops/us          B/op
- * get_concurrentHashtable            2593.6 ± 116.7        ~0
- * get_concurrentHashMap              1830.7 ±  31.5        ~0
- * get_concurrentSkipListMap           202.7 ±  75.6        ~0
- * get_synchronizedHashMap               9.2 ±   0.8        ~0
+ * get_concurrentHashtable            2647.5 ±  74.6        ~0
+ * get_concurrentHashMap              1837.1 ±  44.0        ~0
+ * get_concurrentSkipListMap           203.4 ±  36.3        ~0
+ * get_synchronizedHashMap               9.3 ±   0.4        ~0
  *
- * getOrCreate_concurrentHashtable    2580.6 ±  28.3        ~0
- * getOrCreate_concurrentHashMap      1823.9 ±  50.2        ~0
- * getOrCreate_synchronizedHashMap       9.3 ±   0.7        ~0
+ * getOrCreate_concurrentHashtable    2589.7 ±  60.9        ~0
+ * getOrCreate_concurrentHashMap      1842.2 ±  33.1        ~0
+ * getOrCreate_synchronizedHashMap       9.3 ±   0.8        ~0
  * }
        
  *
  * 
        Allocation is measured with {@code -prof gc}. Every arm is allocation-free: the keys are
@@ -58,14 +58,14 @@
  * 
        Key findings:
  *
  * 
          
- *   
        • {@code ConcurrentHashtable} is ~40% faster than {@code ConcurrentHashMap} on {@code get}
- *       (2593.6 vs 1830.7 ops/us); avoids the hash-to-segment translation CHM pays even on its fast
+ *   
        • {@code ConcurrentHashtable} is ~44% faster than {@code ConcurrentHashMap} on {@code get}
+ *       (2647.5 vs 1837.1 ops/us); avoids the hash-to-segment translation CHM pays even on its fast
  *       path.
  *   
        • {@code ConcurrentSkipListMap} is ~9× slower than {@code ConcurrentHashMap} — tree traversal
- *       cost is high even under lock-free CAS. Its error bar is wide (±75.6 on a 202.7 mean), so
+ *       cost is high even under lock-free CAS. Its error bar is wide (±36.3 on a 203.4 mean), so
  *       treat that multiple as approximate.
- *   
        • Synchronized {@code HashMap} is roughly 280× slower than {@code ConcurrentHashtable} (9.2
- *       vs 2593.6 ops/us) — lock contention across eight threads on a single monitor, which is what
+ *   
        • Synchronized {@code HashMap} is roughly 280× slower than {@code ConcurrentHashtable} (9.3
+ *       vs 2647.5 ops/us) — lock contention across eight threads on a single monitor, which is what
  *       this benchmark isolates. Type-profile pollution is not a factor: the keys are {@code
  *       String}, a final class, so these lookups devirtualize by exact type and never consult the
  *       polluted profile.
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
index 921e4fd4fd2..24f12bcd7ec 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
@@ -45,24 +45,26 @@
  *
  * 
          {@code
  * Benchmark                               ops/us          B/op
- * get_support                         2730.0 ±  33.4        ~0
- * get_concurrentHashtable             2653.0 ±  81.6        ~0
- * get_concurrentHashMap               1665.0 ±  24.2        ~0
- * get_concurrentSkipListMap            187.3 ±  40.1        ~0
- * get_synchronizedHashMap                9.3 ±   0.4        ~0
+ * get_support                         2731.5 ±  48.2        ~0
+ * get_concurrentHashtable             2545.5 ± 322.8        ~0
+ * get_concurrentHashMap               1668.2 ±  34.2        ~0
+ * get_concurrentSkipListMap            182.2 ±  43.8        ~0
+ * get_synchronizedHashMap                9.4 ±   0.5        ~0
  *
- * getOrCreate_support                 2597.4 ± 221.5        ~0
- * getOrCreate_concurrentHashMap       1647.1 ±  75.5        ~0
- * getOrCreate_concurrentHashtable     1397.0 ±  46.5        ~0
- * getOrCreate_concurrentSkipListMap    179.1 ±  40.0        ~0
- * getOrCreate_synchronizedHashMap        9.3 ±   0.4        ~0
+ * getOrCreate_support                 2497.8 ±  73.0        ~0
+ * getOrCreate_concurrentHashMap       1655.5 ±  62.4        ~0
+ * getOrCreate_concurrentHashtable     1397.3 ±  72.7        ~0
+ * getOrCreate_concurrentSkipListMap    178.2 ±  31.7        ~0
+ * getOrCreate_synchronizedHashMap        8.7 ±   2.4      12.0
  * }
          
  *
  * 
          The {@link Key2} wrapper is not allocated in the measured code. Every map arm reports
- * ≈0 B/op under {@code -prof gc}, despite the source constructing a {@code Key2} per lookup.
- * LogCompilation confirms the mechanism: C2 emits {@code eliminate_allocation} for {@code Key2},
- * because the never-taken {@code computeIfAbsent} branch is pruned as {@code unstable_if}, which
- * removes the only store of the key and leaves it provably non-escaping.
+ * ≈0 B/op under {@code -prof gc}, despite the source constructing a {@code Key2} per lookup. The
+ * lone exception is {@code getOrCreate_synchronizedHashMap} at 12 B/op, whose throughput interval
+ * (±2.4 on an 8.7 mean) is too wide to read anything into; not investigated. LogCompilation
+ * confirms the mechanism: C2 emits {@code eliminate_allocation} for {@code Key2}, because the
+ * never-taken {@code computeIfAbsent} branch is pruned as {@code unstable_if}, which removes the
+ * only store of the key and leaves it provably non-escaping.
  *
  * 
          That is a property of this workload, not of the code, and it would not survive in production.
  * Pruning is possible only because {@code @Setup} installs every key before warmup, so the absent
@@ -77,19 +79,19 @@
  * 
          Key findings:
  *
  * 
            
- *   
          • {@code Support} and {@code ConcurrentHashtable} are the two fastest on {@code get} (2730.0
- *       and 2653.0 ops/us), ~60% ahead of {@code ConcurrentHashMap} (1665.0). Since the {@code
+ *   
          • {@code Support} and {@code ConcurrentHashtable} are the two fastest on {@code get} (2731.5
+ *       and 2545.5 ops/us), 53-64% ahead of {@code ConcurrentHashMap} (1668.2). Since the {@code
  *       Key2} allocation is eliminated in all three (see above), that lead is the two-level hash
  *       lookup rather than allocation.
- *   
          • On {@code getOrCreate} the ordering inverts: {@code ConcurrentHashMap} (1647.1) overtakes
- *       {@code ConcurrentHashtable} (1397.0), whose own {@code getOrCreate} is roughly half its
- *       {@code get}. {@code Support} holds up (2597.4). Not root-caused here — the write-path
+ *   
          • On {@code getOrCreate} the ordering inverts: {@code ConcurrentHashMap} (1655.5) overtakes
+ *       {@code ConcurrentHashtable} (1397.3), whose own {@code getOrCreate} is roughly half its
+ *       {@code get}. {@code Support} holds up (2497.8). Not root-caused here — the write-path
  *       re-check is the obvious suspect, but it is not measured.
  *   
          • {@code Support} edges {@code D2} on both paths, consistent with its primitive {@code int}
  *       K2 field avoiding boxing inside the entry match on the write-path re-check.
  *   
          • {@code ConcurrentSkipListMap} is ~9× slower than {@code ConcurrentHashMap} due to tree
- *       traversal, though its error bar is wide (±40.1 on a 187.3 mean).
- *   
          • Synchronized {@code HashMap} is roughly 290× slower than the fastest options (9.3 vs 2730.0
+ *       traversal, though its error bar is wide (±43.8 on a 182.2 mean).
+ *   
          • Synchronized {@code HashMap} is roughly 290× slower than the fastest options (9.4 vs 2731.5
  *       ops/us) — lock contention across eight threads on a single monitor, the same magnitude seen
  *       in {@link ThreadSafeMapD1Benchmark}. Type-profile pollution is not a factor: {@code Key2}
  *       is a final class built at the call site, so C2 has an exact type and devirtualizes without

From 49a25acef92620e3a6c5b10451410204b3386f0f Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Tue, 6 Oct 2026 10:11:52 -0400
Subject: [PATCH 38/52] Correct dispatch and API claims in benchmark javadocs

Describe warmUpHashDispatch as broadening shared profiles rather than
making them megamorphic (receiver counts still drive C2's inlining),
drop the segment-based explanation for ConcurrentHashMap, and remove the
false claim that ConcurrentSkipListMap lacks computeIfAbsent.

Co-Authored-By: Claude Opus 5.5 
---
 .../trace/api/TagMapAccessBenchmark.java        |  8 ++++----
 .../java/datadog/trace/util/BenchmarkUtils.java | 17 ++++++++++-------
 .../trace/util/ThreadSafeMapD1Benchmark.java    |  3 +--
 .../trace/util/ThreadSafeMapD2Benchmark.java    |  4 ++--
 4 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java
index dae425c6b66..4d0c4e3cb7a 100644
--- a/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/api/TagMapAccessBenchmark.java
@@ -78,13 +78,13 @@
  * so it has no {@code Object}-typed dispatch site to pollute — it is insulated by construction,
  * whatever the JIT decides. {@code HashMap} does have such sites: {@code getNode} calls {@code
  * hashCode()}/{@code equals()} on an {@code Object}-declared key. Erasure makes that one bytecode
- * index serve every map in the application, so in production it sees many receiver types and is
- * genuinely megamorphic — which is the condition {@code warmUpHashDispatch} reproduces here. Even
- * so, {@code get}/{@code put} inline into a caller holding a statically exact {@code String}, so C2
+ * index serve every map in the application, so in production it sees many receiver types — the
+ * condition {@code warmUpHashDispatch} approximates here by broadening that profile. Even so,
+ * {@code get}/{@code put} inline into a caller holding a statically exact {@code String}, so C2
  * sharpens the argument and devirtualizes without consulting that profile. That is a realistic
  * condition rather than an artifact — plenty of production call sites do hand HashMap a statically
  * known key type, and it legitimately gets that benefit. Where the key type is not deducible, the
- * megamorphic profile governs HashMap and still does not reach TagMap.
+ * broadened profile governs HashMap and still does not reach TagMap.
  *
  * 
            Consistent with that, most entries move by a couple of percent with overlapping intervals, and
  * the larger moves go up, which pollution cannot cause.
diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index 5479d1b9cf0..3cd5ca58a17 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -120,13 +120,16 @@ private BenchmarkUtils() {}
    * Comparator}-based call sites.
    *
    * 
            The benchmark's own collection call sites are not invoked here and remain free to specialize
-   * for their receiver types. A call site's recorded type profile isn't threatened by which key
-   * type happens to dominate traffic during or after warmup -- once HotSpot has recorded multiple
-   * receiver types there, it stays megamorphic regardless of later call frequency. The risk this
-   * class guards against is a statically deducible receiver type bypassing the profile entirely:
-   * loading extra classes here defeats class-hierarchy analysis (optimization based on which
-   * implementations are actually loaded), and {@link #distinctEqualCopy}'s {@code DONT_INLINE}
-   * stops the JIT from tracing a decoy key's type back to its origin through static inference.
+   * for their receiver types. This broadens the shared internal profiles; it does not guarantee
+   * that C2 treats them as megamorphic. Receiver counts still matter: C2 can inline both receivers
+   * of a bimorphic profile, or a dominant receiver behind a type guard, and the benchmark's own
+   * warmup traffic keeps adding to those counts until the compile happens -- so a single key type
+   * can come to dominate an internal site this method touched. Only call sites confirmed in the
+   * compiled benchmark (e.g. via {@code -XX:+PrintInlining} or {@code LogCompilation}) should be
+   * described as megamorphic. Loading extra classes here also defeats class-hierarchy analysis
+   * (optimization based on which implementations are actually loaded), and {@link
+   * #distinctEqualCopy}'s {@code DONT_INLINE} stops the JIT from tracing a decoy key's type back to
+   * its origin through static inference.
    */
   public static void warmUpHashDispatch(Blackhole bh) {
     if (!WARMED_UP.compareAndSet(false, true)) {
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
index 9a1d440531d..46f4b65b0a3 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD1Benchmark.java
@@ -59,8 +59,7 @@
  *
  * 
              
  *   
            • {@code ConcurrentHashtable} is ~44% faster than {@code ConcurrentHashMap} on {@code get}
- *       (2647.5 vs 1837.1 ops/us); avoids the hash-to-segment translation CHM pays even on its fast
- *       path.
+ *       (2647.5 vs 1837.1 ops/us).
  *   
            • {@code ConcurrentSkipListMap} is ~9× slower than {@code ConcurrentHashMap} — tree traversal
  *       cost is high even under lock-free CAS. Its error bar is wide (±36.3 on a 203.4 mean), so
  *       treat that multiple as approximate.
diff --git a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
index 24f12bcd7ec..0739e487627 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/ThreadSafeMapD2Benchmark.java
@@ -341,8 +341,8 @@ public Long getOrCreate_concurrentHashMap(SharedState s, ThreadState t) {
   }
 
   /**
-   * get-first pattern for ConcurrentSkipListMap — manual get-then-putIfAbsent since CSLM has no
-   * computeIfAbsent. Two traversals on miss; one on hit.
+   * Get-first pattern for {@link ConcurrentSkipListMap}: hits use one {@code get}; misses also call
+   * {@code putIfAbsent} to handle concurrent insertion.
    */
   @Benchmark
   public Long getOrCreate_concurrentSkipListMap(SharedState s, ThreadState t) {

From 7adb401e705cc3aa2c18e95a2ba65ebac4679602 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Tue, 6 Oct 2026 10:16:58 -0400
Subject: [PATCH 39/52] Narrow HashtableD1Benchmark's setup-allocation finding

The once-per-thread vs once-per-JVM comparison shows the amount of
pollution work doesn't matter; it can't show whether a single pass
affects the trial, since both runs include one.

Co-Authored-By: Claude Opus 5.5 
---
 .../datadog/trace/util/HashtableD1Benchmark.java | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
index 28112a8071c..a9bde5c2610 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
@@ -77,14 +77,14 @@
  * explain a drop on either side. The JDK and machine were held constant, so what remains is
  * uncontrolled run-to-run variation.
  *
- * 
              The other candidate — that {@code warmUpHashDispatch}'s own heavy pre-measurement allocation
- * shifts GC state for the trial — has since been tested and ruled out. Pollution used to run
- * once per thread, so at {@code @Threads(8)} it did eight times the work; it now runs once per JVM.
- * Re-measuring across that 8x reduction moved nothing: {@code update_hashMap} 686.0 to 689.6,
- * {@code update_hashtable} 2770.7 to 2796.3, {@code iterate_hashMap} 19.83 to 19.84, all well
- * inside their intervals. Setup allocation is therefore not what moved these numbers. The
- * relative conclusion (D1 dominates {@code update}, is roughly comparable on {@code add},
- * ties on {@code iterate}) is unchanged throughout.
+ * 
              The other candidate is that {@code warmUpHashDispatch}'s own heavy pre-measurement allocation
+ * shifts GC state for the trial. Pollution used to run once per thread, so at {@code @Threads(8)}
+ * it did eight times the work; it now runs once per JVM. Re-measuring across that 8x reduction
+ * moved nothing: {@code update_hashMap} 686.0 to 689.6, {@code update_hashtable} 2770.7 to 2796.3,
+ * {@code iterate_hashMap} 19.83 to 19.84, all well inside their intervals. So the amount of
+ * setup allocation isn't a factor at this scale; whether a single pass affects the trial at all was
+ * not tested, since both runs include one. The relative conclusion (D1 dominates {@code
+ * update}, is roughly comparable on {@code add}, ties on {@code iterate}) is unchanged throughout.
  *
  * 
              Separately rerun on Zulu 17.0.7 (native AArch64, same machine, pollution wiring unchanged; JMH
  * auto-detected the cheap "compiler" Blackhole mode here, unlike JDK 8, so absolute numbers below

From f41429c08a7939a9b650986448230a6654e711ef Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Tue, 6 Oct 2026 10:51:15 -0400
Subject: [PATCH 40/52] Seed CaseInsensitiveMapBenchmark's lookup keys and
 refresh results

Unseeded per-fork key draws gave each arm a different case and hit/miss
mix. With a fixed sequence, HashMap's apparent ~28% lookup lead over
FlatHashtable disappears (384.7 +/- 81.0 vs 402.6 +/- 11.9), and the
folded-String allocation becomes deterministic. Default to @Fork(5) to
match the documented configuration.

Co-Authored-By: Claude Opus 5.5 
---
 .../util/CaseInsensitiveMapBenchmark.java     | 70 +++++++++----------
 1 file changed, 34 insertions(+), 36 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
index 4d0ef8b479b..0a1bf8d5516 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
@@ -1,8 +1,8 @@
 package datadog.trace.util;
 
 import java.util.HashMap;
+import java.util.Random;
 import java.util.TreeMap;
-import java.util.concurrent.ThreadLocalRandom;
 import java.util.function.Supplier;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.Fork;
@@ -24,48 +24,43 @@
  *       allocation-free (case folded inside hash/matches), value stored unboxed
  * 
            
  *
- * 
            Takeaways. FlatHashtable is ~1.8x the (previously recommended) TreeMap at the same zero
- * allocation, but — see the revised takeaway below — trails HashMap's look-up throughput; its case
- * is the allocation win (no per-look-up folded String, which drives the multi-threaded GC pressure
- * HashMap pays), not a throughput win. The case-insensitive hash is the consistent-for-all-inputs
- * two-way fold ({@link datadog.trace.util.Strings#caseInsensitiveHashCode} — see its note); a
- * cheaper ASCII-only fold would recover a few percent for header-name-only hot paths, deliberately
- * not the default. {@code LOW_LOAD_FACTOR} makes no difference here (the fold, not the probe count,
- * dominates), so the default 0.5 is used.
+ * 
            Takeaways. FlatHashtable is ~2.7x the (previously recommended) TreeMap at the same zero
+ * allocation, and matches HashMap's look-up throughput while avoiding HashMap's per-look-up folded
+ * String. The case-insensitive hash is the consistent-for-all-inputs two-way fold ({@link
+ * datadog.trace.util.Strings#caseInsensitiveHashCode} — see its note); a cheaper ASCII-only fold
+ * would recover a few percent for header-name-only hot paths, deliberately not the default. {@code
+ * LOW_LOAD_FACTOR} makes no measurable difference here, so the default 0.5 is used.
  *
- * 
            Java 17 results (MacBook M1, {@code @Fork(5)}, {@code @Threads(8)}) with the front-loaded
- * {@link BenchmarkUtils#warmUpHashDispatch} pollution design (M ops/s):
+ * 
            Java 17 results (Zulu 17.0.7, MacBook M1, {@code @Fork(5)}, {@code @Threads(8)}, {@code -prof
+ * gc}) with the front-loaded {@link BenchmarkUtils#warmUpHashDispatch} pollution design:
  *
  * 
            {@code
- * create_baseline        25.2    create_flatHashtable    15.3
- * create_hashMap          7.3    create_treeMap           8.4
+ * Benchmark                      M ops/s           B/op
+ * create_baseline                  25.5 ±   1.9    1152
+ * create_flatHashtable             14.8 ±   0.7    1693
+ * create_hashMap                    7.5 ±   0.2    2320
+ * create_treeMap                    8.1 ±   0.8    1840
  *
- * lookup_baseline       2760.8   lookup_flatHashtable    380.3
- * lookup_flatHashtable_lowLoad  430.7  lookup_hashMap    488.5
- * lookup_treeMap         214.7
+ * lookup_baseline                2630.3 ± 128.8      ~0
+ * lookup_flatHashtable            402.6 ±  11.9      ~0
+ * lookup_flatHashtable_lowLoad    411.6 ±  13.8      ~0
+ * lookup_hashMap                  384.7 ±  81.0    22.5
+ * lookup_treeMap                  148.9 ±  42.0      ~0
  * }
            
  *
- * 
            {@code lookup_flatHashtable}/{@code lookup_hashMap}/{@code lookup_treeMap} carry error bars of
- * ~13-17% of their means at {@code @Fork(5)} (down from 44-66% at {@code @Fork(2)}, which wasn't
- * decisive) — tight enough that {@code hashMap}'s lead over {@code flatHashtable} (488.5 vs 380.3,
- * ~28%) is a real, if not perfectly clean-cut, result rather than noise.
+ * 
            Every arm cycles through the same fixed-seed {@code LOOKUP_KEYS} sequence, so all of them see
+ * the same upper/lower-case and hit/miss mix. An earlier run drew the keys unseeded per fork and
+ * showed {@code hashMap} ~28% ahead of {@code flatHashtable}; with the workload held constant that
+ * lead disappears, and {@code hashMap}'s interval (±21% of its mean) comfortably overlaps {@code
+ * flatHashtable}'s (±3%). Throughput is a tie; {@code flatHashtable} is the steadier of the two.
  *
- * 
            Measured with {@code -prof gc}: {@code lookup_hashMap} is the only lookup arm that
- * allocates, at 25.5 ± 4.6 B/op for the folded {@code String}, against ≈0 for {@code
- * flatHashtable}, {@code flatHashtable_lowLoad}, {@code treeMap} and the baseline. That quantifies
- * the allocation win claimed above. The variance is itself informative: {@code toLowerCase()}
- * returns {@code this} when a string is already lower-case, so the figure tracks the mixed-case
- * fraction of the key set rather than a fixed per-lookup cost. Allocation came from a later run
- * than the throughput table, so the two are not a matched pair.
- *
- * 
            Takeaway, revised. {@code HashMap} keyed on {@code toLowerCase()} is faster than {@code
- * FlatHashtable} for this lookup shape, not merely comparable to it as earlier (noisier) runs
- * suggested. {@code FlatHashtable} still wins on allocation — it is the zero-allocation option, and
- * that stays true regardless of the throughput ordering — but the throughput case for it over
- * {@code HashMap} on case-insensitive lookups does not hold up under this rerun. {@code TreeMap}
- * remains the slowest of the three at every fork count measured.
+ * 
            {@code lookup_hashMap} is the only lookup arm that allocates: 22.5 B/op for the folded
+ * {@code String}, which is the allocation win claimed above. {@code toLowerCase()} returns {@code
+ * this} when a string is already lower-case, so the figure reflects this key set's upper-case share
+ * rather than a fixed per-lookup cost; with the seeded keys it is deterministic (±0.001), where the
+ * unseeded run varied ±4.6. {@code TreeMap} remains the slowest look-up.
  */
-@Fork(2)
+@Fork(5)
 @Warmup(iterations = 2)
 @Measurement(iterations = 3)
 @Threads(8)
@@ -89,10 +84,13 @@ static  T init(Supplier supplier) {
             return upperPrefixes;
           });
 
+  // Fixed seed so every arm, in every fork, sees the same case and hit/miss mix -- the upper-case
+  // share drives toLowerCase() allocation and suffix NUM_SUFFIXES is a miss, so an unseeded draw
+  // would give each arm a different workload.
   static final String[] LOOKUP_KEYS =
       init(
           () -> {
-            ThreadLocalRandom curRandom = ThreadLocalRandom.current();
+            Random curRandom = new Random(42);
 
             String[] keys = new String[32];
             for (int i = 0; i < keys.length; ++i) {

From 8493096feed6ad569bd426a5fbe32ac0aec4bbe6 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Tue, 6 Oct 2026 12:55:30 -0400
Subject: [PATCH 41/52] Attribute HashtableD1Benchmark's update ratio to the
 collector, not the JDK

Replace the Java 8-to-17 allocator story with a controlled same-JDK
comparison (Zulu 17, forced FULL_DONTINLINE Blackhole, -prof gc):
disabling EliminateAutoBox changes nothing, while ParallelGC cuts
update_hashMap ~4x, matching JDK 8's default-collector number. The
mechanism is left unattributed since reported gc.time is similar.

Co-Authored-By: Claude Opus 5.5 
---
 .../trace/util/HashtableD1Benchmark.java      | 41 +++++++++++++------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
index a9bde5c2610..19b35f862d3 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java
@@ -108,18 +108,35 @@
  * hashtable's {@code update} and {@code iterate} paths allocate nothing at all, which is the point
  * of the design.
  *
- * 
            Within this single run (so the cross-JDK Blackhole-mode confound doesn't apply to the ratios),
- * {@code update_hashtable} wins by ~4.0x — down from ~14x on JDK 8, because Java 17's allocator/GC
- * absorbs {@code update_hashMap}'s per-call {@code Long} boxing far better than JDK 8 did
- * (update_hashMap itself got ~5x faster; update_hashtable only ~1.5x faster). {@code
- * iterate_hashtable} also now clearly wins (~4.0x), flipping from JDK 8's "wash" — HashMap's {@code
- * entrySet()} iterator does more per-entry work than a modern JIT's allocation improvements erase.
- * {@code add} is the one case that flips the other way: {@code add_hashMap} leads on the means
- * (1658.6 vs 1358.0), though both error bars are wide enough to overlap, so treat that one as
- * undecided rather than a HashMap win. Net takeaway: {@code Hashtable} is a strong substitute for
- * {@code HashMap} particularly for simple counter/tally use cases with a primitive value, where
- * avoiding the per-update boxing allocation pays off even on a JVM with much better allocation
- * handling than JDK 8 had.
+ * 
            Within this run, {@code update_hashtable} wins by ~4.0x and {@code iterate_hashtable} by
+ * ~4.1x. {@code add_hashMap} leads on the means (1658.6 vs 1358.0), but the error bars overlap, so
+ * treat {@code add} as undecided rather than a HashMap win.
+ *
+ * 
            The {@code update} ratio depends heavily on the collector, so it is not a fixed property of
+ * the two structures. A controlled follow-up on the same Zulu 17 build, with the Blackhole mode
+ * forced to {@code FULL_DONTINLINE} for every configuration (separate invocations, so allow ~20%
+ * run-to-run spread):
+ *
+ * 
            {@code
+ * Configuration              update_hashMap   update_hashtable   ratio
+ * G1 (default)                522.4 ±  42.1     1408.0 ±  90.0    2.7x
+ * G1, -XX:-EliminateAutoBox   536.5 ±  24.5     1689.6 ±  46.2    3.1x
+ * -XX:+UseParallelGC          129.2 ±  36.8     1634.2 ±  19.3   12.6x
+ * }
            
+ *
+ * 
            Disabling autobox elimination changes nothing for {@code update_hashMap} (still 24 B/op): the
+ * {@code Long} escapes into the map, so there is no box to eliminate. Switching to ParallelGC cuts
+ * {@code update_hashMap} ~4x while leaving the allocation-free {@code update_hashtable} where it
+ * was, so the collector governs how expensive the per-update box is. The mechanism isn't isolated:
+ * total {@code gc.time} is similar under both collectors (1106 vs 1226 ms) despite ParallelGC
+ * running ~3x as many collections, so reported pause time alone doesn't account for the gap. The
+ * forced Blackhole mode also compresses the ratios relative to the table above (2.7x vs 4.0x on
+ * {@code update}, 1.5x vs 4.1x on {@code iterate}), since every arm pays a non-inlined call per
+ * {@code consume}; the default-mode table is the one to quote.
+ *
+ * 
            Net takeaway: {@code Hashtable} is a strong substitute for {@code HashMap} for simple
+ * counter/tally use cases with a primitive value. Avoiding the per-update boxing allocation wins
+ * under G1, and wins by a much larger margin on a throughput collector.
  */
 @Fork(2)
 @Warmup(iterations = 2)

From a74b563bfd2aa361026d27a9525cd939b5af867c Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Tue, 6 Oct 2026 12:55:32 -0400
Subject: [PATCH 42/52] Refresh CaseInsensitiveMapBenchmark on the merged
 CHA-decoy code

The previous seeded run's hashMap/treeMap arms showed intra-fork swings
consistent with external interference. A clean rerun on the merged code
(with the CHA_DEFEAT decoys) restores HashMap's ~20-30% lookup lead over
FlatHashtable; FlatHashtable's wide interval is one slow-compiling fork.

Co-Authored-By: Claude Opus 5.5 
---
 .../util/CaseInsensitiveMapBenchmark.java     | 42 ++++++++++---------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
index 9e2c75fe4ef..6d9e695d6da 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java
@@ -24,41 +24,43 @@
  *       allocation-free (case folded inside hash/matches), value stored unboxed
  * 
          
  *
- * 
          Takeaways. FlatHashtable is ~2.7x the (previously recommended) TreeMap at the same zero
- * allocation, and matches HashMap's look-up throughput while avoiding HashMap's per-look-up folded
- * String. The case-insensitive hash is the consistent-for-all-inputs two-way fold ({@link
+ * 
          Takeaways. FlatHashtable is ~1.8-2x the (previously recommended) TreeMap at the same
+ * zero allocation, but trails HashMap's look-up throughput by ~20-30%; its case is the allocation
+ * win (no per-look-up folded String), not a throughput win. The case-insensitive hash is the
+ * consistent-for-all-inputs two-way fold ({@link
  * datadog.trace.util.Strings#caseInsensitiveHashCode} — see its note); a cheaper ASCII-only fold
  * would recover a few percent for header-name-only hot paths, deliberately not the default. {@code
- * LOW_LOAD_FACTOR} makes no measurable difference here, so the default 0.5 is used.
+ * LOW_LOAD_FACTOR} makes no consistent difference here, so the default 0.5 is used.
  *
  * 
          Java 17 results (Zulu 17.0.7, MacBook M1, {@code @Fork(5)}, {@code @Threads(8)}, {@code -prof
- * gc}) with the front-loaded {@link BenchmarkUtils#warmUpHashDispatch} pollution design:
+ * gc}) with the front-loaded {@link BenchmarkUtils#warmUpHashDispatch} pollution design and the
+ * {@code CHA_DEFEAT} decoys:
  *
  * 
          {@code
  * Benchmark                      M ops/s           B/op
- * create_baseline                  25.5 ±   1.9    1152
- * create_flatHashtable             14.8 ±   0.7    1693
- * create_hashMap                    7.5 ±   0.2    2320
- * create_treeMap                    8.1 ±   0.8    1840
+ * create_baseline                  25.5 ±   0.4    1152
+ * create_flatHashtable             14.4 ±   0.7    1693
+ * create_hashMap                    7.5 ±   0.3    2320
+ * create_treeMap                    7.9 ±   1.0    1840
  *
- * lookup_baseline                2630.3 ± 128.8      ~0
- * lookup_flatHashtable            402.6 ±  11.9      ~0
- * lookup_flatHashtable_lowLoad    411.6 ±  13.8      ~0
- * lookup_hashMap                  384.7 ±  81.0    22.5
- * lookup_treeMap                  148.9 ±  42.0      ~0
+ * lookup_baseline                2772.0 ±  23.5      ~0
+ * lookup_flatHashtable            385.1 ±  51.5      ~0
+ * lookup_flatHashtable_lowLoad    425.5 ±   6.0      ~0
+ * lookup_hashMap                  502.4 ±   7.9    22.5
+ * lookup_treeMap                  214.6 ±  11.8      ~0
  * }
          
  *
  * 
          Every arm cycles through the same fixed-seed {@code LOOKUP_KEYS} sequence, so all of them see
- * the same upper/lower-case and hit/miss mix. An earlier run drew the keys unseeded per fork and
- * showed {@code hashMap} ~28% ahead of {@code flatHashtable}; with the workload held constant that
- * lead disappears, and {@code hashMap}'s interval (±21% of its mean) comfortably overlaps {@code
- * flatHashtable}'s (±3%). Throughput is a tie; {@code flatHashtable} is the steadier of the two.
+ * the same upper/lower-case and hit/miss mix. {@code lookup_hashMap} is steady across all five
+ * forks (483-509 per iteration). {@code lookup_flatHashtable}'s wide interval comes from one fork
+ * that compiled slow (~295 throughout, against ~380-420 for the other four); excluding it, the gap
+ * to {@code hashMap} is ~20%, and including it ~30%. Either way HashMap leads on throughput.
  *
  * 
          {@code lookup_hashMap} is the only lookup arm that allocates: 22.5 B/op for the folded
  * {@code String}, which is the allocation win claimed above. {@code toLowerCase()} returns {@code
  * this} when a string is already lower-case, so the figure reflects this key set's upper-case share
- * rather than a fixed per-lookup cost; with the seeded keys it is deterministic (±0.001), where the
- * unseeded run varied ±4.6. {@code TreeMap} remains the slowest look-up.
+ * rather than a fixed per-lookup cost; with the seeded keys it is deterministic (±0.001), where an
+ * earlier unseeded run varied ±4.6. {@code TreeMap} remains the slowest look-up.
  */
 @Fork(5)
 @Warmup(iterations = 2)

From 184fbfd6b39f8c4f20ed96653de2602ff14138e2 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Fri, 25 Sep 2026 17:04:04 -0400
Subject: [PATCH 43/52] Add BenchmarkUtils.warmUpArms to restore cross-arm
 profile pollution

JMH forks a fresh JVM per @Benchmark method, so each fork's C2 compiles
against a maximally specialized view: one arm's receiver types, one
arm's branch outcomes. warmUpArms drives every registered arm through
many shuffled-order passes before measurement, restoring the two-sided
branch profile a fork removes.

Arm0/Arm1/Arm2 adapters overload on standard java.util.function shapes
(Function, ToLongFunction, Predicate, Consumer, their Bi* counterparts,
plus TriFunction/TriConsumer for two-@State-parameter arms), so a
method reference or state-capturing lambda to an existing @Benchmark
method coerces in directly, and overload resolution picks the
non-boxing primitive interface whenever the referenced method returns
an unboxed primitive.

Coverage is declared, not inferred: registering fewer arms than the
benchmark class's matching-shape @Benchmark methods fails loudly at
setup time rather than silently warming a subset.

Jira: APMLP-1870

Co-Authored-By: Claude Sonnet 5 
---
 .../datadog/trace/util/BenchmarkUtils.java    | 297 +++++++++++++++++-
 1 file changed, 294 insertions(+), 3 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index 3cd5ca58a17..6d41e6439e4 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -1,16 +1,27 @@
 package datadog.trace.util;
 
+import java.lang.reflect.Method;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.BiPredicate;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.ToLongBiFunction;
+import java.util.function.ToLongFunction;
+import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.CompilerControl;
 import org.openjdk.jmh.infra.Blackhole;
 
@@ -65,9 +76,20 @@
  * 
          Such a helper should drive the arms in a shuffled order rather than a fixed round-robin, since
  * compilation can trigger part-way through a warmup and would otherwise see whichever arm dominates
  * that point in the sequence; a regular pattern also gives the hardware branch predictor an
- * unrealistically easy time. Seeding the shuffle keeps it irregular but reproducible. A {@code
- * warmUpArms} along those lines is planned as a follow-on, because adopting it changes benchmark
- * setup and requires re-measuring whatever adopts it.
+ * unrealistically easy time. Seeding the shuffle keeps it irregular but reproducible.
+ *
+ * 
          {@link #warmUpArms} is that helper, for the third gap above. An "arm" is one path through a
+ * benchmark class -- ordinarily one {@code @Benchmark} method, or one outcome of a method that has
+ * several (e.g. {@code getOrCreate}'s hit and miss paths). Rather than ask the author to wrap each
+ * arm in a bespoke adapter, {@code arm(...)} is overloaded on the standard {@code
+ * java.util.function} shapes that real benchmark methods already have, mirroring how {@code
+ * TagMapFuzzTest.MapAction}'s {@code BasicAction}/{@code BasicReturningAction} adapters bridge
+ * differently-shaped map operations onto one common interface -- so a state-capturing lambda like
+ * {@code () -> update_hashMap(state)} or an unbound method reference like {@code
+ * Foo::create_hashMap} coerces directly into one of the overloads, and overload resolution picks
+ * the non-boxing primitive interface (e.g. {@code ToLongFunction}) over the boxing one ({@code
+ * Function}) whenever the referenced method returns an unboxed primitive, so driving a {@code
+ * long}-returning arm never boxes it.
  */
 public final class BenchmarkUtils {
   private BenchmarkUtils() {}
@@ -284,4 +306,273 @@ private static Object distinctEqualCopy(Object key) {
       return key;
     }
   }
+
+  // ---- warmUpArms: restores cross-arm profile pollution that per-method forking removes ----
+
+  /**
+   * Fixed seed for the warmup shuffle, so the order is irregular (unlike round-robin) but
+   * reproducible from run to run.
+   */
+  private static final long ARM_SHUFFLE_SEED = 0x415233A2C0FFEEL;
+
+  /** One no-argument arm -- see {@link #warmUpArms(Object, Blackhole, Arm0[])}. */
+  public abstract static class Arm0 {
+    abstract void invokeAndConsume(B bench, Blackhole bh);
+  }
+
+  /**
+   * One single-{@code @State}-parameter arm -- see {@link #warmUpArms(Object, Object, Blackhole,
+   * Arm1[])}.
+   */
+  public abstract static class Arm1 {
+    abstract void invokeAndConsume(B bench, S state, Blackhole bh);
+  }
+
+  /**
+   * One two-{@code @State}-parameter arm -- see {@link #warmUpArms(Object, Object, Object,
+   * Blackhole, Arm2[])}.
+   */
+  public abstract static class Arm2 {
+    abstract void invokeAndConsume(B bench, S1 s1, S2 s2, Blackhole bh);
+  }
+
+  /** Two-{@code @State}-parameter function, for arms shaped like {@code R method(S1 s1, S2 s2)}. */
+  @FunctionalInterface
+  public interface TriFunction {
+    R apply(A a, B b, C c);
+  }
+
+  /**
+   * Three-argument consumer, for arms shaped like {@code void method(S state, Blackhole bh)} that
+   * consume the {@link Blackhole} themselves rather than returning a value for {@code warmUpArms}
+   * to consume on their behalf.
+   */
+  @FunctionalInterface
+  public interface TriConsumer {
+    void accept(A a, B b, C c);
+  }
+
+  // Arm0 adapters -- an unbound reference to a no-arg @Benchmark method, or a state-capturing
+  // lambda like `() -> update_hashMap(state)`, coerces directly into one of these.
+
+  public static  Arm0 arm(Function fn) {
+    return new Arm0() {
+      @Override
+      void invokeAndConsume(B bench, Blackhole bh) {
+        bh.consume(fn.apply(bench));
+      }
+    };
+  }
+
+  public static  Arm0 arm(ToLongFunction fn) {
+    return new Arm0() {
+      @Override
+      void invokeAndConsume(B bench, Blackhole bh) {
+        bh.consume(fn.applyAsLong(bench));
+      }
+    };
+  }
+
+  public static  Arm0 arm(Predicate fn) {
+    return new Arm0() {
+      @Override
+      void invokeAndConsume(B bench, Blackhole bh) {
+        bh.consume(fn.test(bench));
+      }
+    };
+  }
+
+  public static  Arm0 arm(Consumer fn) {
+    return new Arm0() {
+      @Override
+      void invokeAndConsume(B bench, Blackhole bh) {
+        fn.accept(bench);
+      }
+    };
+  }
+
+  // Arm1 adapters -- an unbound reference to a single-@State-parameter @Benchmark method coerces
+  // directly into one of these (a trailing Blackhole parameter on the real method is unaffected,
+  // since JMH passes that in separately from the arm's own signature).
+
+  public static  Arm1 arm(BiFunction fn) {
+    return new Arm1() {
+      @Override
+      void invokeAndConsume(B bench, S state, Blackhole bh) {
+        bh.consume(fn.apply(bench, state));
+      }
+    };
+  }
+
+  public static  Arm1 arm(ToLongBiFunction fn) {
+    return new Arm1() {
+      @Override
+      void invokeAndConsume(B bench, S state, Blackhole bh) {
+        bh.consume(fn.applyAsLong(bench, state));
+      }
+    };
+  }
+
+  public static  Arm1 arm(BiPredicate fn) {
+    return new Arm1() {
+      @Override
+      void invokeAndConsume(B bench, S state, Blackhole bh) {
+        bh.consume(fn.test(bench, state));
+      }
+    };
+  }
+
+  public static  Arm1 arm(BiConsumer fn) {
+    return new Arm1() {
+      @Override
+      void invokeAndConsume(B bench, S state, Blackhole bh) {
+        fn.accept(bench, state);
+      }
+    };
+  }
+
+  // A method reference to a void method that also takes a trailing Blackhole (e.g. `void
+  // iterate_hashtable(D1State s, Blackhole bh)`) is a 3-arg unbound reference (receiver, state,
+  // Blackhole), so it needs its own adapter rather than fitting BiConsumer above.
+  public static  Arm1 arm(TriConsumer fn) {
+    return new Arm1() {
+      @Override
+      void invokeAndConsume(B bench, S state, Blackhole bh) {
+        fn.accept(bench, state, bh);
+      }
+    };
+  }
+
+  // Arm2 adapters -- an unbound reference to a two-@State-parameter @Benchmark method (e.g.
+  // getOrCreate(SharedState, ThreadState)) coerces directly into this one.
+
+  public static  Arm2 arm(TriFunction fn) {
+    return new Arm2() {
+      @Override
+      void invokeAndConsume(B bench, S1 s1, S2 s2, Blackhole bh) {
+        bh.consume(fn.apply(bench, s1, s2));
+      }
+    };
+  }
+
+  /**
+   * Drives every registered no-argument arm {@link #WARM_UP_ITERATIONS} times, in a shuffled order
+   * re-permuted each pass, so that by the time a {@code @Benchmark} method is measured, C2 has
+   * already compiled against the whole class's mix of outcomes rather than the one path the fork is
+   * about to isolate.
+   *
+   * 
          Coverage is declared, not inferred: register one arm per {@code @Benchmark} method, and one
+   * per interesting outcome of a method that has several (e.g. {@code getOrCreate}'s hit and miss
+   * paths) -- registering only the hit path is worse than not calling this at all, since it turns a
+   * known gap into a silent one. {@link #warmUpArms(Object, Blackhole, Arm0[])} can only check that
+   * at least as many arms were registered as there are matching {@code @Benchmark} methods; it
+   * cannot tell that every outcome of a multi-outcome method was covered.
+   */
+  @SafeVarargs
+  public static  void warmUpArms(B bench, Blackhole bh, Arm0... arms) {
+    checkArmCoverage(bench.getClass(), 0, arms.length);
+    int[] order = identityOrder(arms.length);
+    Random random = new Random(ARM_SHUFFLE_SEED);
+    for (int pass = 0; pass < WARM_UP_ITERATIONS; ++pass) {
+      shuffle(order, random);
+      for (int idx : order) {
+        arms[idx].invokeAndConsume(bench, bh);
+      }
+    }
+  }
+
+  /**
+   * As {@link #warmUpArms(Object, Blackhole, Arm0[])}, for arms taking one {@code @State}
+   * parameter.
+   */
+  @SafeVarargs
+  public static  void warmUpArms(B bench, S state, Blackhole bh, Arm1... arms) {
+    checkArmCoverage(bench.getClass(), 1, arms.length);
+    int[] order = identityOrder(arms.length);
+    Random random = new Random(ARM_SHUFFLE_SEED);
+    for (int pass = 0; pass < WARM_UP_ITERATIONS; ++pass) {
+      shuffle(order, random);
+      for (int idx : order) {
+        arms[idx].invokeAndConsume(bench, state, bh);
+      }
+    }
+  }
+
+  /**
+   * As {@link #warmUpArms(Object, Blackhole, Arm0[])}, for arms taking two {@code @State}
+   * parameters.
+   */
+  @SafeVarargs
+  public static  void warmUpArms(
+      B bench, S1 s1, S2 s2, Blackhole bh, Arm2... arms) {
+    checkArmCoverage(bench.getClass(), 2, arms.length);
+    int[] order = identityOrder(arms.length);
+    Random random = new Random(ARM_SHUFFLE_SEED);
+    for (int pass = 0; pass < WARM_UP_ITERATIONS; ++pass) {
+      shuffle(order, random);
+      for (int idx : order) {
+        arms[idx].invokeAndConsume(bench, s1, s2, bh);
+      }
+    }
+  }
+
+  private static int[] identityOrder(int length) {
+    int[] order = new int[length];
+    for (int i = 0; i < length; ++i) {
+      order[i] = i;
+    }
+    return order;
+  }
+
+  /** Fisher-Yates shuffle, in place, no per-pass allocation. */
+  private static void shuffle(int[] order, Random random) {
+    for (int i = order.length - 1; i > 0; --i) {
+      int j = random.nextInt(i + 1);
+      int tmp = order[i];
+      order[i] = order[j];
+      order[j] = tmp;
+    }
+  }
+
+  /**
+   * Coarse "did you forget to register an arm" check: counts the {@code @Benchmark} methods
+   * declared on {@code benchClass} whose parameter list -- ignoring a trailing {@link Blackhole}
+   * parameter, which JMH supplies independently of an arm's own state parameters -- has {@code
+   * expectedStateArity} parameters, and fails loudly if fewer arms than that were registered.
+   *
+   * 
          This is deliberately weak: it can't tell which method is missing, and it can't tell whether
+   * a multi-outcome method had every outcome driven, only that some arm exists for its
+   * shape. Widening this to per-method, per-outcome tracking is a real follow-on once this coarse
+   * form proves insufficient in practice, not before.
+   */
+  private static void checkArmCoverage(
+      Class benchClass, int expectedStateArity, int registeredArms) {
+    int declaredBenchmarks = 0;
+    for (Method m : benchClass.getDeclaredMethods()) {
+      if (!m.isAnnotationPresent(Benchmark.class)) {
+        continue;
+      }
+      Class[] params = m.getParameterTypes();
+      int arity = params.length;
+      if (arity > 0 && params[arity - 1] == Blackhole.class) {
+        --arity;
+      }
+      if (arity == expectedStateArity) {
+        ++declaredBenchmarks;
+      }
+    }
+    if (registeredArms < declaredBenchmarks) {
+      throw new AssertionError(
+          "warmUpArms: "
+              + benchClass.getSimpleName()
+              + " declares "
+              + declaredBenchmarks
+              + " @Benchmark method(s) with "
+              + expectedStateArity
+              + " state parameter(s), but only "
+              + registeredArms
+              + " arm(s) were registered for that shape -- coverage is declared, not inferred; see"
+              + " the Arm0/Arm1/Arm2 javadoc.");
+    }
+  }
 }

From c7b01bf3edf614bd91a4a565665cda97896bebf9 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Fri, 25 Sep 2026 17:32:39 -0400
Subject: [PATCH 44/52] Rename Arm/arm/warmUpArms to Bench/bench/warmUp,
 collapse to boxing-tolerant overloads

Drops the primitive-specific overloads (ToLongFunction/Predicate/
ToLongBiFunction/BiPredicate) in favor of a single Function/Consumer
pair per arity plus TriFunction/TriConsumer for the arity-driven
shapes; a primitive result boxes on the way into Blackhole.consume,
which is warmup-only overhead off the measured path. Also adds a
fully reflective warmUp(Class, Blackhole) overload that discovers
every @Benchmark method on a class and drives it without any
registration, for benchmarks where one arm per method is enough.

Co-Authored-By: Claude Sonnet 5 
---
 .../datadog/trace/util/BenchmarkUtils.java    | 324 +++++++++++-------
 1 file changed, 205 insertions(+), 119 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index 6d41e6439e4..2f72a830f9a 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -1,10 +1,13 @@
 package datadog.trace.util;
 
+import java.lang.reflect.Constructor;
 import java.lang.reflect.Method;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
@@ -15,14 +18,11 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
-import java.util.function.BiPredicate;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import java.util.function.Predicate;
-import java.util.function.ToLongBiFunction;
-import java.util.function.ToLongFunction;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.infra.Blackhole;
 
 /**
@@ -78,18 +78,20 @@
  * that point in the sequence; a regular pattern also gives the hardware branch predictor an
  * unrealistically easy time. Seeding the shuffle keeps it irregular but reproducible.
  *
- * 
          {@link #warmUpArms} is that helper, for the third gap above. An "arm" is one path through a
+ * 
          {@link #warmUp} is that helper, for the third gap above. A "bench" is one path through a
  * benchmark class -- ordinarily one {@code @Benchmark} method, or one outcome of a method that has
  * several (e.g. {@code getOrCreate}'s hit and miss paths). Rather than ask the author to wrap each
- * arm in a bespoke adapter, {@code arm(...)} is overloaded on the standard {@code
- * java.util.function} shapes that real benchmark methods already have, mirroring how {@code
- * TagMapFuzzTest.MapAction}'s {@code BasicAction}/{@code BasicReturningAction} adapters bridge
- * differently-shaped map operations onto one common interface -- so a state-capturing lambda like
- * {@code () -> update_hashMap(state)} or an unbound method reference like {@code
- * Foo::create_hashMap} coerces directly into one of the overloads, and overload resolution picks
- * the non-boxing primitive interface (e.g. {@code ToLongFunction}) over the boxing one ({@code
- * Function}) whenever the referenced method returns an unboxed primitive, so driving a {@code
- * long}-returning arm never boxes it.
+ * one in a bespoke adapter, {@code bench(...)} is overloaded on a small, deliberately
+ * boxing-tolerant set of {@code java.util.function} shapes that real benchmark methods already
+ * have, mirroring how {@code TagMapFuzzTest.MapAction}'s {@code BasicAction}/{@code
+ * BasicReturningAction} adapters bridge differently-shaped map operations onto one common interface
+ * -- so a state-capturing lambda like {@code () -> update_hashMap(state)} or an unbound method
+ * reference like {@code Foo::create_hashMap} coerces directly into one of the overloads. The
+ * wrapper always takes the {@link Blackhole} and consumes whatever the wrapped method returns,
+ * boxing a primitive result if there is one; that box is warmup-only overhead, off the measured
+ * path, so there is nothing to gain by avoiding it. For classes where one arm per
+ * {@code @Benchmark} method is enough, {@link #warmUp(Class, Blackhole)} skips registration
+ * entirely and discovers everything reflectively.
  */
 public final class BenchmarkUtils {
   private BenchmarkUtils() {}
@@ -307,44 +309,46 @@ private static Object distinctEqualCopy(Object key) {
     }
   }
 
-  // ---- warmUpArms: restores cross-arm profile pollution that per-method forking removes ----
+  // ---- warmUp: restores cross-arm profile pollution that per-method forking removes ----
 
   /**
    * Fixed seed for the warmup shuffle, so the order is irregular (unlike round-robin) but
    * reproducible from run to run.
    */
-  private static final long ARM_SHUFFLE_SEED = 0x415233A2C0FFEEL;
+  private static final long SHUFFLE_SEED = 0x415233A2C0FFEEL;
 
-  /** One no-argument arm -- see {@link #warmUpArms(Object, Blackhole, Arm0[])}. */
-  public abstract static class Arm0 {
+  /** One no-argument bench -- see {@link #warmUp(Object, Blackhole, Bench0[])}. */
+  public abstract static class Bench0 {
     abstract void invokeAndConsume(B bench, Blackhole bh);
   }
 
   /**
-   * One single-{@code @State}-parameter arm -- see {@link #warmUpArms(Object, Object, Blackhole,
-   * Arm1[])}.
+   * One single-{@code @State}-parameter bench -- see {@link #warmUp(Object, Object, Blackhole,
+   * Bench1[])}.
    */
-  public abstract static class Arm1 {
+  public abstract static class Bench1 {
     abstract void invokeAndConsume(B bench, S state, Blackhole bh);
   }
 
   /**
-   * One two-{@code @State}-parameter arm -- see {@link #warmUpArms(Object, Object, Object,
-   * Blackhole, Arm2[])}.
+   * One two-{@code @State}-parameter bench -- see {@link #warmUp(Object, Object, Object, Blackhole,
+   * Bench2[])}.
    */
-  public abstract static class Arm2 {
+  public abstract static class Bench2 {
     abstract void invokeAndConsume(B bench, S1 s1, S2 s2, Blackhole bh);
   }
 
-  /** Two-{@code @State}-parameter function, for arms shaped like {@code R method(S1 s1, S2 s2)}. */
+  /**
+   * Two-{@code @State}-parameter function, for benches shaped like {@code R method(S1 s1, S2 s2)}.
+   */
   @FunctionalInterface
   public interface TriFunction {
     R apply(A a, B b, C c);
   }
 
   /**
-   * Three-argument consumer, for arms shaped like {@code void method(S state, Blackhole bh)} that
-   * consume the {@link Blackhole} themselves rather than returning a value for {@code warmUpArms}
+   * Three-argument consumer, for benches shaped like {@code void method(S state, Blackhole bh)}
+   * that consume the {@link Blackhole} themselves rather than returning a value for {@code warmUp}
    * to consume on their behalf.
    */
   @FunctionalInterface
@@ -352,11 +356,12 @@ public interface TriConsumer {
     void accept(A a, B b, C c);
   }
 
-  // Arm0 adapters -- an unbound reference to a no-arg @Benchmark method, or a state-capturing
-  // lambda like `() -> update_hashMap(state)`, coerces directly into one of these.
+  // Bench0 adapters -- an unbound reference to a no-arg @Benchmark method, or a state-capturing
+  // lambda like `() -> update_hashMap(state)`, coerces directly into one of these. A primitive
+  // return boxes here, but that box is warmup-only overhead, off the measured path.
 
-  public static  Arm0 arm(Function fn) {
-    return new Arm0() {
+  public static  Bench0 bench(Function fn) {
+    return new Bench0() {
       @Override
       void invokeAndConsume(B bench, Blackhole bh) {
         bh.consume(fn.apply(bench));
@@ -364,26 +369,8 @@ void invokeAndConsume(B bench, Blackhole bh) {
     };
   }
 
-  public static  Arm0 arm(ToLongFunction fn) {
-    return new Arm0() {
-      @Override
-      void invokeAndConsume(B bench, Blackhole bh) {
-        bh.consume(fn.applyAsLong(bench));
-      }
-    };
-  }
-
-  public static  Arm0 arm(Predicate fn) {
-    return new Arm0() {
-      @Override
-      void invokeAndConsume(B bench, Blackhole bh) {
-        bh.consume(fn.test(bench));
-      }
-    };
-  }
-
-  public static  Arm0 arm(Consumer fn) {
-    return new Arm0() {
+  public static  Bench0 bench(Consumer fn) {
+    return new Bench0() {
       @Override
       void invokeAndConsume(B bench, Blackhole bh) {
         fn.accept(bench);
@@ -391,12 +378,12 @@ void invokeAndConsume(B bench, Blackhole bh) {
     };
   }
 
-  // Arm1 adapters -- an unbound reference to a single-@State-parameter @Benchmark method coerces
+  // Bench1 adapters -- an unbound reference to a single-@State-parameter @Benchmark method coerces
   // directly into one of these (a trailing Blackhole parameter on the real method is unaffected,
-  // since JMH passes that in separately from the arm's own signature).
+  // since JMH passes that in separately from the bench's own signature).
 
-  public static  Arm1 arm(BiFunction fn) {
-    return new Arm1() {
+  public static  Bench1 bench(BiFunction fn) {
+    return new Bench1() {
       @Override
       void invokeAndConsume(B bench, S state, Blackhole bh) {
         bh.consume(fn.apply(bench, state));
@@ -404,26 +391,8 @@ void invokeAndConsume(B bench, S state, Blackhole bh) {
     };
   }
 
-  public static  Arm1 arm(ToLongBiFunction fn) {
-    return new Arm1() {
-      @Override
-      void invokeAndConsume(B bench, S state, Blackhole bh) {
-        bh.consume(fn.applyAsLong(bench, state));
-      }
-    };
-  }
-
-  public static  Arm1 arm(BiPredicate fn) {
-    return new Arm1() {
-      @Override
-      void invokeAndConsume(B bench, S state, Blackhole bh) {
-        bh.consume(fn.test(bench, state));
-      }
-    };
-  }
-
-  public static  Arm1 arm(BiConsumer fn) {
-    return new Arm1() {
+  public static  Bench1 bench(BiConsumer fn) {
+    return new Bench1() {
       @Override
       void invokeAndConsume(B bench, S state, Blackhole bh) {
         fn.accept(bench, state);
@@ -434,8 +403,8 @@ void invokeAndConsume(B bench, S state, Blackhole bh) {
   // A method reference to a void method that also takes a trailing Blackhole (e.g. `void
   // iterate_hashtable(D1State s, Blackhole bh)`) is a 3-arg unbound reference (receiver, state,
   // Blackhole), so it needs its own adapter rather than fitting BiConsumer above.
-  public static  Arm1 arm(TriConsumer fn) {
-    return new Arm1() {
+  public static  Bench1 bench(TriConsumer fn) {
+    return new Bench1() {
       @Override
       void invokeAndConsume(B bench, S state, Blackhole bh) {
         fn.accept(bench, state, bh);
@@ -443,11 +412,11 @@ void invokeAndConsume(B bench, S state, Blackhole bh) {
     };
   }
 
-  // Arm2 adapters -- an unbound reference to a two-@State-parameter @Benchmark method (e.g.
+  // Bench2 adapters -- an unbound reference to a two-@State-parameter @Benchmark method (e.g.
   // getOrCreate(SharedState, ThreadState)) coerces directly into this one.
 
-  public static  Arm2 arm(TriFunction fn) {
-    return new Arm2() {
+  public static  Bench2 bench(TriFunction fn) {
+    return new Bench2() {
       @Override
       void invokeAndConsume(B bench, S1 s1, S2 s2, Blackhole bh) {
         bh.consume(fn.apply(bench, s1, s2));
@@ -456,62 +425,179 @@ void invokeAndConsume(B bench, S1 s1, S2 s2, Blackhole bh) {
   }
 
   /**
-   * Drives every registered no-argument arm {@link #WARM_UP_ITERATIONS} times, in a shuffled order
-   * re-permuted each pass, so that by the time a {@code @Benchmark} method is measured, C2 has
-   * already compiled against the whole class's mix of outcomes rather than the one path the fork is
-   * about to isolate.
+   * Drives every registered no-argument bench {@link #WARM_UP_ITERATIONS} times, in a shuffled
+   * order re-permuted each pass, so that by the time a {@code @Benchmark} method is measured, C2
+   * has already compiled against the whole class's mix of outcomes rather than the one path the
+   * fork is about to isolate.
    *
-   * 
          Coverage is declared, not inferred: register one arm per {@code @Benchmark} method, and one
-   * per interesting outcome of a method that has several (e.g. {@code getOrCreate}'s hit and miss
-   * paths) -- registering only the hit path is worse than not calling this at all, since it turns a
-   * known gap into a silent one. {@link #warmUpArms(Object, Blackhole, Arm0[])} can only check that
-   * at least as many arms were registered as there are matching {@code @Benchmark} methods; it
-   * cannot tell that every outcome of a multi-outcome method was covered.
+   * 
          Coverage is declared, not inferred: register one bench per {@code @Benchmark} method, and
+   * one per interesting outcome of a method that has several (e.g. {@code getOrCreate}'s hit and
+   * miss paths) -- registering only the hit path is worse than not calling this at all, since it
+   * turns a known gap into a silent one. {@link #warmUp(Object, Blackhole, Bench0[])} can only
+   * check that at least as many benches were registered as there are matching {@code @Benchmark}
+   * methods; it cannot tell that every outcome of a multi-outcome method was covered.
    */
   @SafeVarargs
-  public static  void warmUpArms(B bench, Blackhole bh, Arm0... arms) {
-    checkArmCoverage(bench.getClass(), 0, arms.length);
-    int[] order = identityOrder(arms.length);
-    Random random = new Random(ARM_SHUFFLE_SEED);
+  public static  void warmUp(B bench, Blackhole bh, Bench0... benches) {
+    checkCoverage(bench.getClass(), 0, benches.length);
+    int[] order = identityOrder(benches.length);
+    Random random = new Random(SHUFFLE_SEED);
     for (int pass = 0; pass < WARM_UP_ITERATIONS; ++pass) {
       shuffle(order, random);
       for (int idx : order) {
-        arms[idx].invokeAndConsume(bench, bh);
+        benches[idx].invokeAndConsume(bench, bh);
       }
     }
   }
 
   /**
-   * As {@link #warmUpArms(Object, Blackhole, Arm0[])}, for arms taking one {@code @State}
+   * As {@link #warmUp(Object, Blackhole, Bench0[])}, for benches taking one {@code @State}
    * parameter.
    */
   @SafeVarargs
-  public static  void warmUpArms(B bench, S state, Blackhole bh, Arm1... arms) {
-    checkArmCoverage(bench.getClass(), 1, arms.length);
-    int[] order = identityOrder(arms.length);
-    Random random = new Random(ARM_SHUFFLE_SEED);
+  public static  void warmUp(B bench, S state, Blackhole bh, Bench1... benches) {
+    checkCoverage(bench.getClass(), 1, benches.length);
+    int[] order = identityOrder(benches.length);
+    Random random = new Random(SHUFFLE_SEED);
     for (int pass = 0; pass < WARM_UP_ITERATIONS; ++pass) {
       shuffle(order, random);
       for (int idx : order) {
-        arms[idx].invokeAndConsume(bench, state, bh);
+        benches[idx].invokeAndConsume(bench, state, bh);
       }
     }
   }
 
   /**
-   * As {@link #warmUpArms(Object, Blackhole, Arm0[])}, for arms taking two {@code @State}
+   * As {@link #warmUp(Object, Blackhole, Bench0[])}, for benches taking two {@code @State}
    * parameters.
    */
   @SafeVarargs
-  public static  void warmUpArms(
-      B bench, S1 s1, S2 s2, Blackhole bh, Arm2... arms) {
-    checkArmCoverage(bench.getClass(), 2, arms.length);
-    int[] order = identityOrder(arms.length);
-    Random random = new Random(ARM_SHUFFLE_SEED);
+  public static  void warmUp(
+      B bench, S1 s1, S2 s2, Blackhole bh, Bench2... benches) {
+    checkCoverage(bench.getClass(), 2, benches.length);
+    int[] order = identityOrder(benches.length);
+    Random random = new Random(SHUFFLE_SEED);
+    for (int pass = 0; pass < WARM_UP_ITERATIONS; ++pass) {
+      shuffle(order, random);
+      for (int idx : order) {
+        benches[idx].invokeAndConsume(bench, s1, s2, bh);
+      }
+    }
+  }
+
+  /**
+   * Fully automatic variant of {@link #warmUp(Object, Blackhole, Bench0[])}: reflectively discovers
+   * every {@code @Benchmark} method on {@code benchClass}, builds {@code benchClass} and any
+   * {@code @State} parameter types via their public no-arg constructor (as JMH itself requires of
+   * every {@code @State} class), runs any no-arg {@code @Setup} methods it finds on those state
+   * instances, then drives every discovered method the same shuffled way as the explicit overloads.
+   *
+   * 
          This trades precision for zero effort: coverage is exactly "every {@code @Benchmark} method
+   * on the class", so a method with several outcomes (e.g. {@code getOrCreate}'s hit and miss) only
+   * gets whichever outcome its one reflectively-built state happens to produce, and a
+   * {@code @Setup} that takes a {@link Blackhole} or {@code Control} parameter is skipped rather
+   * than guessed at. Reach for the explicit {@code warmUp} overloads above when a benchmark needs
+   * more than one outcome driven or a state built some other way; use this one when a single arm
+   * per method, reflectively constructed, is enough.
+   */
+  public static void warmUp(Class benchClass, Blackhole bh) {
+    Object bench = newInstance(benchClass);
+    Map, Object> stateInstances = new HashMap<>();
+    List invocations = new ArrayList<>();
+    for (Method m : benchClass.getDeclaredMethods()) {
+      if (!m.isAnnotationPresent(Benchmark.class)) {
+        continue;
+      }
+      m.setAccessible(true);
+      Class[] paramTypes = m.getParameterTypes();
+      Object[] args = new Object[paramTypes.length];
+      for (int i = 0; i < paramTypes.length; ++i) {
+        args[i] =
+            paramTypes[i] == Blackhole.class
+                ? bh
+                : stateInstances.computeIfAbsent(paramTypes[i], BenchmarkUtils::newStateInstance);
+      }
+      invocations.add(new ReflectiveInvocation(bench, m, args));
+    }
+    if (invocations.isEmpty()) {
+      throw new AssertionError(
+          "warmUp: " + benchClass.getSimpleName() + " declares no @Benchmark methods");
+    }
+    int[] order = identityOrder(invocations.size());
+    Random random = new Random(SHUFFLE_SEED);
     for (int pass = 0; pass < WARM_UP_ITERATIONS; ++pass) {
       shuffle(order, random);
       for (int idx : order) {
-        arms[idx].invokeAndConsume(bench, s1, s2, bh);
+        invocations.get(idx).invokeAndConsume(bh);
+      }
+    }
+  }
+
+  /**
+   * One reflectively-bound {@code @Benchmark} method call, for {@link #warmUp(Class, Blackhole)}.
+   */
+  private static final class ReflectiveInvocation {
+    private final Object bench;
+    private final Method method;
+    private final Object[] args;
+
+    ReflectiveInvocation(Object bench, Method method, Object[] args) {
+      this.bench = bench;
+      this.method = method;
+      this.args = args;
+    }
+
+    void invokeAndConsume(Blackhole bh) {
+      try {
+        Object result = method.invoke(bench, args);
+        if (method.getReturnType() != void.class) {
+          bh.consume(result);
+        }
+      } catch (ReflectiveOperationException e) {
+        throw new AssertionError("warmUp: failed to invoke " + method, e);
+      }
+    }
+  }
+
+  /** Builds a {@code @State} instance via its public no-arg constructor and runs its setup. */
+  private static Object newStateInstance(Class type) {
+    Object state = newInstance(type);
+    runSetups(state);
+    return state;
+  }
+
+  private static Object newInstance(Class type) {
+    try {
+      Constructor ctor = type.getDeclaredConstructor();
+      ctor.setAccessible(true);
+      return ctor.newInstance();
+    } catch (ReflectiveOperationException e) {
+      throw new AssertionError(
+          "warmUp: "
+              + type.getName()
+              + " has no public no-arg constructor -- JMH requires one on every @State class; use"
+              + " the explicit warmUp(bench, state, Blackhole, Bench1...) overload instead if this"
+              + " type needs to be built another way.",
+          e);
+    }
+  }
+
+  /**
+   * Runs every no-arg {@code @Setup} method declared on {@code state}. A {@code @Setup} method
+   * taking a {@link Blackhole} or {@code Control} parameter is skipped silently -- {@link
+   * #warmUp(Class, Blackhole)} is the best-effort, zero-configuration path; a benchmark that needs
+   * one should use the explicit {@code warmUp} overloads instead.
+   */
+  private static void runSetups(Object state) {
+    for (Method m : state.getClass().getDeclaredMethods()) {
+      if (!m.isAnnotationPresent(Setup.class) || m.getParameterCount() != 0) {
+        continue;
+      }
+      m.setAccessible(true);
+      try {
+        m.invoke(state);
+      } catch (ReflectiveOperationException e) {
+        throw new AssertionError("warmUp: failed to run @Setup " + m + " on " + state, e);
       }
     }
   }
@@ -535,18 +621,18 @@ private static void shuffle(int[] order, Random random) {
   }
 
   /**
-   * Coarse "did you forget to register an arm" check: counts the {@code @Benchmark} methods
+   * Coarse "did you forget to register a bench" check: counts the {@code @Benchmark} methods
    * declared on {@code benchClass} whose parameter list -- ignoring a trailing {@link Blackhole}
-   * parameter, which JMH supplies independently of an arm's own state parameters -- has {@code
-   * expectedStateArity} parameters, and fails loudly if fewer arms than that were registered.
+   * parameter, which JMH supplies independently of a bench's own state parameters -- has {@code
+   * expectedStateArity} parameters, and fails loudly if fewer benches than that were registered.
    *
    * 
          This is deliberately weak: it can't tell which method is missing, and it can't tell whether
-   * a multi-outcome method had every outcome driven, only that some arm exists for its
+   * a multi-outcome method had every outcome driven, only that some bench exists for its
    * shape. Widening this to per-method, per-outcome tracking is a real follow-on once this coarse
    * form proves insufficient in practice, not before.
    */
-  private static void checkArmCoverage(
-      Class benchClass, int expectedStateArity, int registeredArms) {
+  private static void checkCoverage(
+      Class benchClass, int expectedStateArity, int registeredBenches) {
     int declaredBenchmarks = 0;
     for (Method m : benchClass.getDeclaredMethods()) {
       if (!m.isAnnotationPresent(Benchmark.class)) {
@@ -561,18 +647,18 @@ private static void checkArmCoverage(
         ++declaredBenchmarks;
       }
     }
-    if (registeredArms < declaredBenchmarks) {
+    if (registeredBenches < declaredBenchmarks) {
       throw new AssertionError(
-          "warmUpArms: "
+          "warmUp: "
               + benchClass.getSimpleName()
               + " declares "
               + declaredBenchmarks
               + " @Benchmark method(s) with "
               + expectedStateArity
               + " state parameter(s), but only "
-              + registeredArms
-              + " arm(s) were registered for that shape -- coverage is declared, not inferred; see"
-              + " the Arm0/Arm1/Arm2 javadoc.");
+              + registeredBenches
+              + " bench(es) were registered for that shape -- coverage is declared, not inferred;"
+              + " see the Bench0/Bench1/Bench2 javadoc.");
     }
   }
 }

From ec7bfa7163170b5beb1d8b4d65e480aec02ba56d Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Fri, 25 Sep 2026 17:57:15 -0400
Subject: [PATCH 45/52] Add WarmUpEscapeAnalysisDemoBenchmark as evidence that
 BenchmarkUtils.warmUp works

Canonical case: a computeIfAbsent-guarded composite-key lookup where
ThreadSafeMapD2Benchmark's javadoc already documents that a one-sided
branch profile lets C2 prune the miss branch as unstable_if and
scalar-replace the composite key, understating real allocation. Both
@Benchmark arms here call the identical getOrCreate bytecode on an
always-hit lookup; the only difference is whether warmUp primes that
same method with one genuine miss arm, on scratch data, before
measurement. Measured effect: 0 B/op with no priming vs 24 B/op (the
composite key's real size) with it, ~20% slower -- confirming warmUp
restores the production-realistic profile.

Co-Authored-By: Claude Sonnet 5 
---
 .../WarmUpEscapeAnalysisDemoBenchmark.java    | 138 ++++++++++++++++++
 1 file changed, 138 insertions(+)
 create mode 100644 internal-api/src/jmh/java/datadog/trace/util/WarmUpEscapeAnalysisDemoBenchmark.java

diff --git a/internal-api/src/jmh/java/datadog/trace/util/WarmUpEscapeAnalysisDemoBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/WarmUpEscapeAnalysisDemoBenchmark.java
new file mode 100644
index 00000000000..a7615abae44
--- /dev/null
+++ b/internal-api/src/jmh/java/datadog/trace/util/WarmUpEscapeAnalysisDemoBenchmark.java
@@ -0,0 +1,138 @@
+package datadog.trace.util;
+
+import static java.util.concurrent.TimeUnit.MICROSECONDS;
+
+import datadog.trace.util.ThreadSafeMapD2Benchmark.Key2;
+import java.util.concurrent.ConcurrentHashMap;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+/**
+ * Demonstrates that {@link BenchmarkUtils#warmUp} closes the branch-profile gap that {@link
+ * ThreadSafeMapD2Benchmark}'s javadoc documents but does not fix: with every key pre-installed by
+ * {@code @Setup}, {@code getOrCreate}'s absent-key branch never runs, so C2 prunes it as an {@code
+ * unstable_if} and scalar-replaces the {@link Key2} allocation that a real, two-sided profile would
+ * force onto the heap.
+ *
+ * 
          Both {@code @Benchmark} methods call the exact same {@link #getOrCreate} bytecode on an
+ * always-hit lookup; the only difference is whether {@link ThreadState}'s
+ * {@code @Setup(Level.Trial)} drives that same method through a genuine miss arm first, via {@link
+ * BenchmarkUtils#warmUp}, on a scratch map before measurement starts. The branch's
+ * method-data-object lives per call site (bci), not per map instance, so priming it on scratch data
+ * pollutes the profile the measured call sees without touching the measured map's contents.
+ *
+ * 
          Java 8 results ({@code -prof gc}), {@code pollute} is the only thing that differs between the
+ * two rows -- same call site, same always-hit measured lookup:
+ *
+ * 
          {@code
+ * pollute   ops/us    B/op     gc.count
+ * false      750.0     ≈0         ≈0     <- unstable_if prunes the miss branch, Key2 scalar-replaced
+ * true       600.5     24.0      763     <- two-sided profile, Key2 allocated on every lookup
+ * }
          
+ *
+ * 
          {@code false} reproduces the artificially flattering result {@link ThreadSafeMapD2Benchmark}'s
+ * javadoc documents. {@code true} shows a single {@link BenchmarkUtils#warmUp} call, priming one
+ * real miss arm before measurement, is enough to restore the allocation and the ~20% slower,
+ * production-realistic throughput -- with no change to the measured method or the measured map.
+ */
+@Fork(2)
+@Warmup(iterations = 2)
+@Measurement(iterations = 3)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(MICROSECONDS)
+@Threads(8)
+public class WarmUpEscapeAnalysisDemoBenchmark {
+
+  static final int N_KEYS = 64;
+  static final String[] SOURCE_K1 = new String[N_KEYS];
+  static final Integer[] SOURCE_K2 = new Integer[N_KEYS];
+
+  static {
+    for (int i = 0; i < N_KEYS; ++i) {
+      SOURCE_K1[i] = "key-" + i;
+      SOURCE_K2[i] = i * 31 + 17;
+    }
+  }
+
+  /** The one call site whose branch profile this benchmark is about; both arms share it. */
+  static Long getOrCreate(ConcurrentHashMap map, Key2 key) {
+    Long existing = map.get(key);
+    if (existing != null) {
+      return existing;
+    }
+    return map.computeIfAbsent(key, k -> 0L);
+  }
+
+  /** Shared, pre-populated table -- the measured lookups always hit. */
+  @State(Scope.Benchmark)
+  public static class SharedState {
+    ConcurrentHashMap map;
+
+    @Setup(Level.Trial)
+    public void setUp() {
+      map = new ConcurrentHashMap<>();
+      for (int i = 0; i < N_KEYS; ++i) {
+        // Plain put, deliberately not routed through getOrCreate: this is the condition
+        // ThreadSafeMapD2Benchmark's javadoc documents as artificially flattering the benchmark.
+        map.put(new Key2(SOURCE_K1[i], SOURCE_K2[i]), (long) i);
+      }
+    }
+  }
+
+  @State(Scope.Thread)
+  public static class ThreadState {
+    /**
+     * Off arm: reproduces the one-sided profile documented in ThreadSafeMapD2Benchmark. On arm:
+     * primes the real getOrCreate branch with a genuine miss before measurement.
+     */
+    @Param({"false", "true"})
+    boolean pollute;
+
+    int cursor;
+    int missCounter;
+    final ConcurrentHashMap scratch = new ConcurrentHashMap<>();
+    final Key2 scratchHitKey = new Key2("scratch-hit", -1);
+
+    @Setup(Level.Trial)
+    public void warmUp(Blackhole bh) {
+      if (!pollute) {
+        return;
+      }
+      getOrCreate(scratch, scratchHitKey); // install the one key the hit arm below will always find
+      BenchmarkUtils.warmUp(
+          this,
+          bh,
+          BenchmarkUtils.bench((ThreadState t) -> getOrCreate(t.scratch, t.scratchHitKey)),
+          BenchmarkUtils.bench((ThreadState t) -> getOrCreate(t.scratch, t.freshMissKey())));
+    }
+
+    /** A key guaranteed never to have been looked up before, so this always takes the miss path. */
+    Key2 freshMissKey() {
+      return new Key2("scratch-miss-" + (missCounter++), -2);
+    }
+
+    int next() {
+      int i = cursor;
+      cursor = (i + 1) & (N_KEYS - 1);
+      return i;
+    }
+  }
+
+  @Benchmark
+  public Long getOrCreate_concurrentHashMap(SharedState s, ThreadState t) {
+    int i = t.next();
+    return getOrCreate(s.map, new Key2(SOURCE_K1[i], SOURCE_K2[i]));
+  }
+}

From bf02bd71c2385d0190815c9b3e07e717864e6169 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Tue, 6 Oct 2026 15:21:45 -0400
Subject: [PATCH 46/52] Correct BenchmarkUtils.warmUp and demo benchmark
 javadocs

The class doc still said branch profiles were unaddressed. Also fix
claims that didn't match the code: a zero-arg lambda example that can't
fit Function, the demo's 'both @Benchmark methods' (there is one,
with two @Param arms), TriFunction's arity, which @Setup methods the
reflective path skips, and its never-set @Param fields.

Co-Authored-By: Claude Opus 5.5 
---
 .../datadog/trace/util/BenchmarkUtils.java    | 89 +++++++++----------
 .../WarmUpEscapeAnalysisDemoBenchmark.java    | 25 +++---
 2 files changed, 57 insertions(+), 57 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index 2f72a830f9a..0a780521894 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -37,8 +37,8 @@
  * another way, JMH forks to eliminate cross-benchmark profile pollution, but profile pollution
  * is the production condition.
  *
- * 
          That surfaces three ways. This class addresses the first two partially and the third not at
- * all:
+ * 
          That surfaces three ways. This class addresses the first two partially, via {@link
+ * #warmUpHashDispatch}, and the third via {@link #warmUp}:
  *
  * 
            
  *   
          • Receiver-type profiles collapse toward a single key type, so shared JDK dispatch
@@ -47,8 +47,9 @@
  *   
          • Class-hierarchy analysis sees only the implementations one arm happens to load, so
  *       C2 can devirtualize calls a real application leaves polymorphic. Loading the decoy
  *       collections widens the hierarchy.
- *   
          • Branch profiles go one-sided, because only one arm's outcomes ever occur. Nothing
- *       here addresses that.
+ *   
          • Branch profiles go one-sided, because only one arm's outcomes ever occur. {@link
+ *       #warmUp} drives every registered arm, and each outcome registered for it, before
+ *       measurement.
  * 
          
  *
  * 
          {@link Blackhole} is orthogonal to all of this. It stops the JIT proving a result is dead, and
@@ -69,29 +70,26 @@
  * HashtableD2Benchmark} for the contrasting shape, where {@code merge} keeps the present/absent
  * decision inside the callee and leaves no caller-visible branch to prune.
  *
- * 
          The more complete approach is to exercise every benchmark arm during setup, across each of its
- * outcomes, so that each fork's profiles reflect the whole class rather than the one arm it is
- * about to measure. That is what restores the mix the fork removed.
+ * 
          {@link #warmUp} exercises every benchmark arm during setup, across each of its outcomes, so
+ * that each fork's profiles reflect the whole class rather than the one arm it is about to measure.
+ * It drives the arms in a shuffled order rather than a fixed round-robin, since compilation can
+ * trigger part-way through a warmup and would otherwise see whichever arm dominates that point in
+ * the sequence; a regular pattern also gives the hardware branch predictor an unrealistically easy
+ * time. The shuffle is seeded, so it is irregular but reproducible.
  *
- * 
          Such a helper should drive the arms in a shuffled order rather than a fixed round-robin, since
- * compilation can trigger part-way through a warmup and would otherwise see whichever arm dominates
- * that point in the sequence; a regular pattern also gives the hardware branch predictor an
- * unrealistically easy time. Seeding the shuffle keeps it irregular but reproducible.
- *
- * 
          {@link #warmUp} is that helper, for the third gap above. A "bench" is one path through a
- * benchmark class -- ordinarily one {@code @Benchmark} method, or one outcome of a method that has
- * several (e.g. {@code getOrCreate}'s hit and miss paths). Rather than ask the author to wrap each
- * one in a bespoke adapter, {@code bench(...)} is overloaded on a small, deliberately
- * boxing-tolerant set of {@code java.util.function} shapes that real benchmark methods already
- * have, mirroring how {@code TagMapFuzzTest.MapAction}'s {@code BasicAction}/{@code
- * BasicReturningAction} adapters bridge differently-shaped map operations onto one common interface
- * -- so a state-capturing lambda like {@code () -> update_hashMap(state)} or an unbound method
- * reference like {@code Foo::create_hashMap} coerces directly into one of the overloads. The
- * wrapper always takes the {@link Blackhole} and consumes whatever the wrapped method returns,
- * boxing a primitive result if there is one; that box is warmup-only overhead, off the measured
- * path, so there is nothing to gain by avoiding it. For classes where one arm per
- * {@code @Benchmark} method is enough, {@link #warmUp(Class, Blackhole)} skips registration
- * entirely and discovers everything reflectively.
+ * 
          A "bench" is one path through a benchmark class -- ordinarily one {@code @Benchmark} method,
+ * or one outcome of a method that has several (e.g. {@code getOrCreate}'s hit and miss paths).
+ * Rather than ask the author to wrap each one in a bespoke adapter, {@code bench(...)} is
+ * overloaded on a small, deliberately boxing-tolerant set of {@code java.util.function} shapes that
+ * real benchmark methods already have, mirroring how {@code TagMapFuzzTest.MapAction}'s {@code
+ * BasicAction}/{@code BasicReturningAction} adapters bridge differently-shaped map operations onto
+ * one common interface -- so an unbound method reference like {@code Foo::update_hashMap} or a
+ * state-capturing lambda like {@code (Foo b) -> b.update_hashMap(state)} coerces directly into one
+ * of the overloads. The wrapper always takes the {@link Blackhole} and consumes whatever the
+ * wrapped method returns, boxing a primitive result if there is one; that box is warmup-only
+ * overhead, off the measured path, so there is nothing to gain by avoiding it. For classes where
+ * one arm per {@code @Benchmark} method is enough, {@link #warmUp(Class, Blackhole)} skips
+ * registration entirely and discovers everything reflectively.
  */
 public final class BenchmarkUtils {
   private BenchmarkUtils() {}
@@ -339,7 +337,8 @@ public abstract static class Bench2 {
   }
 
   /**
-   * Two-{@code @State}-parameter function, for benches shaped like {@code R method(S1 s1, S2 s2)}.
+   * Three-argument function: the receiver of an unbound reference to a two-{@code @State} benchmark
+   * method {@code R method(S1 s1, S2 s2)}, plus its two states.
    */
   @FunctionalInterface
   public interface TriFunction {
@@ -357,8 +356,8 @@ public interface TriConsumer {
   }
 
   // Bench0 adapters -- an unbound reference to a no-arg @Benchmark method, or a state-capturing
-  // lambda like `() -> update_hashMap(state)`, coerces directly into one of these. A primitive
-  // return boxes here, but that box is warmup-only overhead, off the measured path.
+  // lambda like `(Foo b) -> b.update_hashMap(state)`, coerces directly into one of these. A
+  // primitive return boxes here, but only during warmup, never on the measured path.
 
   public static  Bench0 bench(Function fn) {
     return new Bench0() {
@@ -379,8 +378,8 @@ void invokeAndConsume(B bench, Blackhole bh) {
   }
 
   // Bench1 adapters -- an unbound reference to a single-@State-parameter @Benchmark method coerces
-  // directly into one of these (a trailing Blackhole parameter on the real method is unaffected,
-  // since JMH passes that in separately from the bench's own signature).
+  // directly into one of these. A method that also takes a trailing Blackhole needs the
+  // TriConsumer adapter below.
 
   public static  Bench1 bench(BiFunction fn) {
     return new Bench1() {
@@ -433,9 +432,9 @@ void invokeAndConsume(B bench, S1 s1, S2 s2, Blackhole bh) {
    * 
          Coverage is declared, not inferred: register one bench per {@code @Benchmark} method, and
    * one per interesting outcome of a method that has several (e.g. {@code getOrCreate}'s hit and
    * miss paths) -- registering only the hit path is worse than not calling this at all, since it
-   * turns a known gap into a silent one. {@link #warmUp(Object, Blackhole, Bench0[])} can only
-   * check that at least as many benches were registered as there are matching {@code @Benchmark}
-   * methods; it cannot tell that every outcome of a multi-outcome method was covered.
+   * turns a known gap into a silent one. This method can only check that at least as many benches
+   * were registered as there are matching {@code @Benchmark} methods; it cannot tell that every
+   * outcome of a multi-outcome method was covered.
    */
   @SafeVarargs
   public static  void warmUp(B bench, Blackhole bh, Bench0... benches) {
@@ -489,16 +488,17 @@ public static  void warmUp(
    * Fully automatic variant of {@link #warmUp(Object, Blackhole, Bench0[])}: reflectively discovers
    * every {@code @Benchmark} method on {@code benchClass}, builds {@code benchClass} and any
    * {@code @State} parameter types via their public no-arg constructor (as JMH itself requires of
-   * every {@code @State} class), runs any no-arg {@code @Setup} methods it finds on those state
+   * every {@code @State} class), runs the no-arg {@code @Setup} methods declared on those state
    * instances, then drives every discovered method the same shuffled way as the explicit overloads.
    *
    * 
          This trades precision for zero effort: coverage is exactly "every {@code @Benchmark} method
    * on the class", so a method with several outcomes (e.g. {@code getOrCreate}'s hit and miss) only
-   * gets whichever outcome its one reflectively-built state happens to produce, and a
-   * {@code @Setup} that takes a {@link Blackhole} or {@code Control} parameter is skipped rather
-   * than guessed at. Reach for the explicit {@code warmUp} overloads above when a benchmark needs
-   * more than one outcome driven or a state built some other way; use this one when a single arm
-   * per method, reflectively constructed, is enough.
+   * gets whichever outcome its one reflectively-built state happens to produce. A {@code @Setup}
+   * method that takes parameters (such as a {@link Blackhole} or {@code Control}) is skipped rather
+   * than guessed at, and {@code @Param} fields are never set, so they keep their Java default
+   * values. Reach for the explicit {@code warmUp} overloads above when a benchmark needs more than
+   * one outcome driven or a state built some other way; use this one when a single arm per method,
+   * reflectively constructed, is enough.
    */
   public static void warmUp(Class benchClass, Blackhole bh) {
     Object bench = newInstance(benchClass);
@@ -559,7 +559,7 @@ void invokeAndConsume(Blackhole bh) {
     }
   }
 
-  /** Builds a {@code @State} instance via its public no-arg constructor and runs its setup. */
+  /** Builds a {@code @State} instance via its no-arg constructor and runs its no-arg setups. */
   private static Object newStateInstance(Class type) {
     Object state = newInstance(type);
     runSetups(state);
@@ -583,8 +583,8 @@ private static Object newInstance(Class type) {
   }
 
   /**
-   * Runs every no-arg {@code @Setup} method declared on {@code state}. A {@code @Setup} method
-   * taking a {@link Blackhole} or {@code Control} parameter is skipped silently -- {@link
+   * Runs every no-arg {@code @Setup} method declared on {@code state}, regardless of its {@code
+   * Level}. A {@code @Setup} method that takes parameters is skipped silently, because {@link
    * #warmUp(Class, Blackhole)} is the best-effort, zero-configuration path; a benchmark that needs
    * one should use the explicit {@code warmUp} overloads instead.
    */
@@ -628,8 +628,7 @@ private static void shuffle(int[] order, Random random) {
    *
    * 
          This is deliberately weak: it can't tell which method is missing, and it can't tell whether
    * a multi-outcome method had every outcome driven, only that some bench exists for its
-   * shape. Widening this to per-method, per-outcome tracking is a real follow-on once this coarse
-   * form proves insufficient in practice, not before.
+   * shape.
    */
   private static void checkCoverage(
       Class benchClass, int expectedStateArity, int registeredBenches) {
diff --git a/internal-api/src/jmh/java/datadog/trace/util/WarmUpEscapeAnalysisDemoBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/WarmUpEscapeAnalysisDemoBenchmark.java
index a7615abae44..ce91c90ee96 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/WarmUpEscapeAnalysisDemoBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/WarmUpEscapeAnalysisDemoBenchmark.java
@@ -26,12 +26,12 @@
  * unstable_if} and scalar-replaces the {@link Key2} allocation that a real, two-sided profile would
  * force onto the heap.
  *
- * 
          Both {@code @Benchmark} methods call the exact same {@link #getOrCreate} bytecode on an
- * always-hit lookup; the only difference is whether {@link ThreadState}'s
- * {@code @Setup(Level.Trial)} drives that same method through a genuine miss arm first, via {@link
- * BenchmarkUtils#warmUp}, on a scratch map before measurement starts. The branch's
- * method-data-object lives per call site (bci), not per map instance, so priming it on scratch data
- * pollutes the profile the measured call sees without touching the measured map's contents.
+ * 
          The single {@code @Benchmark} method runs the same {@link #getOrCreate} bytecode on an
+ * always-hit lookup under both values of {@code pollute}. The only difference is whether {@link
+ * ThreadState}'s {@code @Setup(Level.Trial)} first drives that method through a genuine miss, via
+ * {@link BenchmarkUtils#warmUp}, on a scratch map. HotSpot records branch counts per bytecode index
+ * in the method's profile, not per map instance, so priming on scratch data changes the profile the
+ * measured call sees without touching the measured map.
  *
  * 
          Java 8 results ({@code -prof gc}), {@code pollute} is the only thing that differs between the
  * two rows -- same call site, same always-hit measured lookup:
@@ -42,10 +42,10 @@
  * true       600.5     24.0      763     <- two-sided profile, Key2 allocated on every lookup
  * }
      • 
  *
- * 
      {@code false} reproduces the artificially flattering result {@link ThreadSafeMapD2Benchmark}'s
- * javadoc documents. {@code true} shows a single {@link BenchmarkUtils#warmUp} call, priming one
- * real miss arm before measurement, is enough to restore the allocation and the ~20% slower,
- * production-realistic throughput -- with no change to the measured method or the measured map.
+ * 
      {@code false} reproduces the artificially flattering result that {@link
+ * ThreadSafeMapD2Benchmark}'s javadoc documents. {@code true} shows that one {@link
+ * BenchmarkUtils#warmUp} call, priming a real miss before measurement, restores the allocation and
+ * its ~20% throughput cost, with no change to the measured method or map.
  */
 @Fork(2)
 @Warmup(iterations = 2)
@@ -94,8 +94,9 @@ public void setUp() {
   @State(Scope.Thread)
   public static class ThreadState {
     /**
-     * Off arm: reproduces the one-sided profile documented in ThreadSafeMapD2Benchmark. On arm:
-     * primes the real getOrCreate branch with a genuine miss before measurement.
+     * {@code false} reproduces the one-sided profile documented in {@link
+     * ThreadSafeMapD2Benchmark}; {@code true} primes {@link
+     * WarmUpEscapeAnalysisDemoBenchmark#getOrCreate}'s miss branch first.
      */
     @Param({"false", "true"})
     boolean pollute;

From 325b6f282b93d7d1d76f9aca2c67ee83102dc305 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Thu, 8 Oct 2026 14:33:36 -0400
Subject: [PATCH 47/52] Count superclass @Benchmark methods in warmUp's
 coverage check

JMH runs @Setup on a generated Foo_jmhType subclass, so the bench's own
class declares no @Benchmark methods and the check counted zero, letting
under-registration pass silently. Walk the hierarchy, counting an
overridden @Benchmark method once.

Co-Authored-By: Claude Opus 5.5 
---
 .../datadog/trace/util/BenchmarkUtils.java    | 38 ++++++++++++-------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index 0a780521894..e113ba0b342 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -622,9 +622,12 @@ private static void shuffle(int[] order, Random random) {
 
   /**
    * Coarse "did you forget to register a bench" check: counts the {@code @Benchmark} methods
-   * declared on {@code benchClass} whose parameter list -- ignoring a trailing {@link Blackhole}
-   * parameter, which JMH supplies independently of a bench's own state parameters -- has {@code
-   * expectedStateArity} parameters, and fails loudly if fewer benches than that were registered.
+   * declared on {@code benchClass} or its superclasses whose parameter list -- ignoring a trailing
+   * {@link Blackhole} parameter, which JMH supplies independently of a bench's own state parameters
+   * -- has {@code expectedStateArity} parameters, and fails loudly if fewer benches than that were
+   * registered. Superclasses matter because JMH runs {@code @Setup} on a generated {@code
+   * Foo_jmhType} subclass, so {@code this} in a setup method never declares the {@code @Benchmark}
+   * methods itself.
    *
    * 
      This is deliberately weak: it can't tell which method is missing, and it can't tell whether
    * a multi-outcome method had every outcome driven, only that some bench exists for its
@@ -633,17 +636,24 @@ private static void shuffle(int[] order, Random random) {
   private static void checkCoverage(
       Class benchClass, int expectedStateArity, int registeredBenches) {
     int declaredBenchmarks = 0;
-    for (Method m : benchClass.getDeclaredMethods()) {
-      if (!m.isAnnotationPresent(Benchmark.class)) {
-        continue;
-      }
-      Class[] params = m.getParameterTypes();
-      int arity = params.length;
-      if (arity > 0 && params[arity - 1] == Blackhole.class) {
-        --arity;
-      }
-      if (arity == expectedStateArity) {
-        ++declaredBenchmarks;
+    Set seen = new HashSet<>();
+    for (Class c = benchClass; c != null; c = c.getSuperclass()) {
+      for (Method m : c.getDeclaredMethods()) {
+        if (!m.isAnnotationPresent(Benchmark.class)) {
+          continue;
+        }
+        Class[] params = m.getParameterTypes();
+        // An overriding @Benchmark method shadows the one it overrides; count it once.
+        if (!seen.add(m.getName() + Arrays.toString(params))) {
+          continue;
+        }
+        int arity = params.length;
+        if (arity > 0 && params[arity - 1] == Blackhole.class) {
+          --arity;
+        }
+        if (arity == expectedStateArity) {
+          ++declaredBenchmarks;
+        }
       }
     }
     if (registeredBenches < declaredBenchmarks) {

From 84615b778578a7316dcc802b83a9ad792ea512a1 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Thu, 8 Oct 2026 14:33:51 -0400
Subject: [PATCH 48/52] Run the benchmark class's own @Setup in reflective
 warmUp

A benchmark class is often an @State with its own @Setup; building it
without running that left its fields null and aborted the trial with an
NPE on the first reflective invocation.

Co-Authored-By: Claude Opus 5.5 
---
 .../src/jmh/java/datadog/trace/util/BenchmarkUtils.java   | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index e113ba0b342..f0cd12ce9ce 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -488,8 +488,9 @@ public static  void warmUp(
    * Fully automatic variant of {@link #warmUp(Object, Blackhole, Bench0[])}: reflectively discovers
    * every {@code @Benchmark} method on {@code benchClass}, builds {@code benchClass} and any
    * {@code @State} parameter types via their public no-arg constructor (as JMH itself requires of
-   * every {@code @State} class), runs the no-arg {@code @Setup} methods declared on those state
-   * instances, then drives every discovered method the same shuffled way as the explicit overloads.
+   * every {@code @State} class), runs the no-arg {@code @Setup} methods declared on the benchmark
+   * instance and those state instances, then drives every discovered method the same shuffled way
+   * as the explicit overloads.
    *
    * 
      This trades precision for zero effort: coverage is exactly "every {@code @Benchmark} method
    * on the class", so a method with several outcomes (e.g. {@code getOrCreate}'s hit and miss) only
@@ -501,7 +502,8 @@ public static  void warmUp(
    * reflectively constructed, is enough.
    */
   public static void warmUp(Class benchClass, Blackhole bh) {
-    Object bench = newInstance(benchClass);
+    // A benchmark class is often an @State itself, with its own @Setup.
+    Object bench = newStateInstance(benchClass);
     Map, Object> stateInstances = new HashMap<>();
     List invocations = new ArrayList<>();
     for (Method m : benchClass.getDeclaredMethods()) {

From a883d9d18343dc150a12b3463fb6709ce8b4e071 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Thu, 8 Oct 2026 14:34:32 -0400
Subject: [PATCH 49/52] Reuse datadog.trace.api.function's TriFunction and
 TriConsumer in warmUp

The nested copies duplicated interfaces of the same shape already in
internal-api.

Co-Authored-By: Claude Opus 5.5 
---
 .../datadog/trace/util/BenchmarkUtils.java    | 21 ++-----------------
 1 file changed, 2 insertions(+), 19 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index f0cd12ce9ce..f7b5552c271 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -1,5 +1,7 @@
 package datadog.trace.util;
 
+import datadog.trace.api.function.TriConsumer;
+import datadog.trace.api.function.TriFunction;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
@@ -336,25 +338,6 @@ public abstract static class Bench2 {
     abstract void invokeAndConsume(B bench, S1 s1, S2 s2, Blackhole bh);
   }
 
-  /**
-   * Three-argument function: the receiver of an unbound reference to a two-{@code @State} benchmark
-   * method {@code R method(S1 s1, S2 s2)}, plus its two states.
-   */
-  @FunctionalInterface
-  public interface TriFunction {
-    R apply(A a, B b, C c);
-  }
-
-  /**
-   * Three-argument consumer, for benches shaped like {@code void method(S state, Blackhole bh)}
-   * that consume the {@link Blackhole} themselves rather than returning a value for {@code warmUp}
-   * to consume on their behalf.
-   */
-  @FunctionalInterface
-  public interface TriConsumer {
-    void accept(A a, B b, C c);
-  }
-
   // Bench0 adapters -- an unbound reference to a no-arg @Benchmark method, or a state-capturing
   // lambda like `(Foo b) -> b.update_hashMap(state)`, coerces directly into one of these. A
   // primitive return boxes here, but only during warmup, never on the measured path.

From 5c80c7f0e66bd732e2fb732e6b4820c37946f035 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Thu, 8 Oct 2026 14:40:03 -0400
Subject: [PATCH 50/52] Drop the demo's scratch keys after warmUp and refresh
 on JDK 17

Each miss pass inserted a fresh key into a per-thread scratch map that
stayed live through measurement, so pollute=true carried ~400k extra
entries the false arm didn't. Clear it after warmUp, and replace the
JDK 8 table with a Zulu 17 run: 0 vs 24 B/op, ~28% throughput cost.

Co-Authored-By: Claude Opus 5.5 
---
 .../util/WarmUpEscapeAnalysisDemoBenchmark.java  | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/WarmUpEscapeAnalysisDemoBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/WarmUpEscapeAnalysisDemoBenchmark.java
index ce91c90ee96..5cfd937e913 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/WarmUpEscapeAnalysisDemoBenchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/WarmUpEscapeAnalysisDemoBenchmark.java
@@ -33,19 +33,20 @@
  * in the method's profile, not per map instance, so priming on scratch data changes the profile the
  * measured call sees without touching the measured map.
  *
- * 
      Java 8 results ({@code -prof gc}), {@code pollute} is the only thing that differs between the
- * two rows -- same call site, same always-hit measured lookup:
+ * 
      Java 17 results (Zulu 17.0.7, MacBook M1, {@code @Fork(2)}, {@code @Threads(8)}, {@code -prof
+ * gc}). {@code pollute} is the only thing that differs between the two rows -- same call site, same
+ * always-hit measured lookup:
  *
  * 
      {@code
- * pollute   ops/us    B/op     gc.count
- * false      750.0     ≈0         ≈0     <- unstable_if prunes the miss branch, Key2 scalar-replaced
- * true       600.5     24.0      763     <- two-sided profile, Key2 allocated on every lookup
+ * pollute   ops/us           B/op  gc.count
+ * false     1610.2 ± 82.4    ≈0      ≈0    <- miss branch pruned (unstable_if), Key2 elided
+ * true      1154.7 ± 66.8    24.0    692   <- two-sided profile; Key2 allocated per lookup
  * }
      
  *
  * 
      {@code false} reproduces the artificially flattering result that {@link
  * ThreadSafeMapD2Benchmark}'s javadoc documents. {@code true} shows that one {@link
  * BenchmarkUtils#warmUp} call, priming a real miss before measurement, restores the allocation and
- * its ~20% throughput cost, with no change to the measured method or map.
+ * its ~28% throughput cost, with no change to the measured method or map.
  */
 @Fork(2)
 @Warmup(iterations = 2)
@@ -117,6 +118,9 @@ public void warmUp(Blackhole bh) {
           bh,
           BenchmarkUtils.bench((ThreadState t) -> getOrCreate(t.scratch, t.scratchHitKey)),
           BenchmarkUtils.bench((ThreadState t) -> getOrCreate(t.scratch, t.freshMissKey())));
+      // Every miss inserted a fresh key; drop them so only the branch profile, not a larger live
+      // heap, separates this arm from pollute=false.
+      scratch.clear();
     }
 
     /** A key guaranteed never to have been looked up before, so this always takes the miss path. */

From d97f0764e4933d1ae14c17246ecd4d3f49a49290 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Tue, 13 Oct 2026 08:22:34 -0400
Subject: [PATCH 51/52] Attribute the FlatHashtable dispatch numbers to a
 one-off measurement

The J12 rubric text said this PR measured FlatHashtableIteratorBenchmark, but
that benchmark is not part of the PR; the 37.9 -> 18.1 M ops/s numbers came
from an uncommitted experiment.

Co-Authored-By: Claude Opus 5.5 
---
 .agents/skills/perf-review/references/checks.md | 2 +-
 .agents/skills/perf-review/references/guide.md  | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.agents/skills/perf-review/references/checks.md b/.agents/skills/perf-review/references/checks.md
index c09dd30f821..978e811229f 100644
--- a/.agents/skills/perf-review/references/checks.md
+++ b/.agents/skills/perf-review/references/checks.md
@@ -51,7 +51,7 @@ Refines the universal checks with JVM mechanics. Quarantined here, for the Java
 - **J9 — `Objects.hash(...)` varargs / boxing hash on a hot path → `HashingUtils`** *(refines #1)*. The allocation is specific to the **varargs/boxing forms**: `Objects.hash(a, b, …)` allocates an `Object[]` per call and **boxes every primitive** arg; same for boxing primitives into a `new Object[]{…}` (or `Arrays.hashCode` over such an array). Per-span tag/key building, or a hot value object's `hashCode()` built this way, → a guaranteed per-call allocation + boxing. Fix: `datadog.trace.util.HashingUtils` — primitive `hash(long/int/boolean/char/…)` overloads (no boxing), `hash(Object,Object)` and `hash(int,int)` combiners (no array); for >2 fields fold pairwise through `hash(int,int)` (there is no varargs form, by design). flag-with-confidence for the varargs/boxing form — SEV-2/3. **Do NOT flag allocation-free combines** — a hand-rolled `31*h + Long.hashCode(x)` / `31*h + intField`, or `Arrays.hashCode` over an *existing primitive array*, allocates nothing (`HashingUtils` is itself 31-based); flagging them would recommend replacing already-correct code.
 - **J10 — hot-path `String.format` / string munging → `Strings` (+ `SubSequence`)** *(refines #2)*. `String.format` parses the format string, boxes its args, and allocates on every call — never on a hot path; hand-rolled case-conversion, class/resource-name munging, blank-checks, and truncation recomputed per call qualify too. Fix: `datadog.trace.util.Strings` — allocation-aware `replace`/`truncate(CharSequence)`/`isBlank`/`getResourceName`/`getClassName`/…; for **transient substring compares** prefer a `SubSequence` view (J7); for plain assembly, direct concatenation beats `format`. flag-with-confidence for `String.format` on a hot path; flag-as-measure for borderline munging — SEV-2/3.
 - **J11 — composite / multi-dimensional key maps on a hot path → `Hashtable` / `ConcurrentHashtable`** *(refines #1, #3)*. `Map>` nesting, or a `HashMap` keyed by a composite key (client-side stats, per-`(service, operation, …)` aggregation), allocates nested maps + `Entry` objects + boxes keys on the hot aggregation path. Fix: `datadog.trace.util.Hashtable` (single-threaded, composite-key D1/D2 tables — landed) or `datadog.trace.util.ConcurrentHashtable` (lock-free concurrent, **coming**) — positional composite keys, fewer allocations. flag-as-measure — SEV-2/3.
-- **J12 — Benchmark validity: is the dispatch under test actually profile-dependent?** *(methodology check on a submitted benchmark, not a production hot-path finding — refines the flag-as-measure posture in J2)*. JMH forks a fresh JVM per benchmark method, so each arm is compiled in a process that has only ever executed that one arm — leaving shared `hashCode()`/`equals()`/`compareTo` call sites unrealistically clean compared with production, where erasure makes them serve every map in the application. This only affects the measurement when the receiver type is **not statically deducible at the benchmark's own call site**: if the key is a final class (`String`) or an exact type from an allocation, C2 devirtualizes on that proof and never consults the profile, so pollution is a no-op that only adds setup allocation. Note that one key class per benchmark is normal and correct — a production `Map` call site is monotyped too; variety at the caller is not what makes the internal site megamorphic. **flag-as-measure** when a Map/Set JMH benchmark's measured operation dispatches on a non-final key type, or reaches a strategy/comparator interface with several loaded implementors, and its `@Setup` does not pollute — fix: call `BenchmarkUtils.warmUpHashDispatch(Blackhole)` from `@Setup(Level.Trial)` (trial level, not per-iteration: it runs 50,000 allocation-heavy passes). PR #12298 measured both sides — `FlatHashtableIteratorBenchmark`'s four-implementor strategy dispatch fell 37.9 → 18.1 M ops/s when its profile was poisoned, while the `String`-keyed map benchmarks in the same module were unaffected, with LogCompilation confirming the dispatch resolved statically.
+- **J12 — Benchmark validity: is the dispatch under test actually profile-dependent?** *(methodology check on a submitted benchmark, not a production hot-path finding — refines the flag-as-measure posture in J2)*. JMH forks a fresh JVM per benchmark method, so each arm is compiled in a process that has only ever executed that one arm — leaving shared `hashCode()`/`equals()`/`compareTo` call sites unrealistically clean compared with production, where erasure makes them serve every map in the application. This only affects the measurement when the receiver type is **not statically deducible at the benchmark's own call site**: if the key is a final class (`String`) or an exact type from an allocation, C2 devirtualizes on that proof and never consults the profile, so pollution is a no-op that only adds setup allocation. Note that one key class per benchmark is normal and correct — a production `Map` call site is monotyped too; variety at the caller is not what makes the internal site megamorphic. **flag-as-measure** when a Map/Set JMH benchmark's measured operation dispatches on a non-final key type, or reaches a strategy/comparator interface with several loaded implementors, and its `@Setup` does not pollute — fix: call `BenchmarkUtils.warmUpHashDispatch(Blackhole)` from `@Setup(Level.Trial)` (trial level, not per-iteration: it runs 50,000 allocation-heavy passes). A one-off measurement made while working on #12298 (not recorded in a committed benchmark) showed both sides — `FlatHashtableIteratorBenchmark`'s four-implementor strategy dispatch fell 37.9 → 18.1 M ops/s when its profile was poisoned, while the `String`-keyed map benchmarks in the same module were unaffected, with LogCompilation confirming the dispatch resolved statically.
 - **Toolkit availability — cite only what exists.** Available today: `Strings`, `SubSequence`, `HashingUtils`, `Hashtable`, `BenchmarkUtils.warmUpHashDispatch(Blackhole)` for JMH setup (all `datadog.trace.util`), `RE2J` (`com.google.re2j`). Coming (name as "coming", don't imply it's present): `ConcurrentHashtable`, `StringIndex` (immutable string set/map), `UTF8BytesString.Cache` (recurring-string interner), wider `IntegerCache` (http-status/port boxing), `DDCache` inlining. **J7–J11 route an *existing* #1/#2/#3 finding to a reusable fix — they are not new flag-triggers. Don't raise a finding you wouldn't have raised anyway; the posture (precision, silent-when-unsure, findings-cap-scales-with-diff — see `SKILL.md`) is unchanged.**
 - **J5 — Cardinality-sensitive aggregator** *(domain-specialized #3)*. Some structures are invisible to the generic "unbounded collection" check because the risk is *cardinality*, not raw size: a config- or user-driven value (tag key, resource name, HTTP URL) feeding a **cardinality-sensitive aggregator** (e.g. the conflating metrics aggregator — each unique label combination = one aggregate; a `maxAggregates` cap bounds OOM but high-cardinality input *thrashes* it: constant eviction, garbled metrics). flag-with-confidence when config/user-driven values reach an aggregator with a per-key budget — **SEV-1** (same class as unbounded memory: correctness + heap impact). Fix: bound the source cardinality before it enters the aggregator, or use sentinel substitution for over-cap values. This surfaced on merged production code more than once in back-test calibration — the capstone pattern where the bot's value concentrates.
 
diff --git a/.agents/skills/perf-review/references/guide.md b/.agents/skills/perf-review/references/guide.md
index d1532495305..3c2b9aaf721 100644
--- a/.agents/skills/perf-review/references/guide.md
+++ b/.agents/skills/perf-review/references/guide.md
@@ -174,7 +174,8 @@ demanding it only adds setup allocation. Having one key class per benchmark is n
 a real `Map` call site is monotyped too. Erasure is what makes the *internal* site
 megamorphic, not variety at the caller.
 
-PR #12298 measured both sides: `FlatHashtableIteratorBenchmark`'s strategy dispatch fell 37.9 → 18.1
+A one-off measurement made while working on #12298 (not recorded in a committed benchmark) showed
+both sides: `FlatHashtableIteratorBenchmark`'s strategy dispatch fell 37.9 → 18.1
 M ops/s (2.1x) when its profile was poisoned, because the strategy is an interface with four loaded
 implementors; the `String`-keyed map benchmarks in the same module were unaffected, and
 LogCompilation confirmed the dispatch was resolved statically. Check that `@Setup` calls

From 85c926a7939b05a3d7680d9ee9d123ef0c90f6c0 Mon Sep 17 00:00:00 2001
From: Douglas Q Hawkins 
Date: Tue, 13 Oct 2026 08:30:55 -0400
Subject: [PATCH 52/52] Apply review wording to BenchmarkUtils and
 HashtableD2Benchmark javadoc

BenchmarkUtils' class javadoc now keeps the fork, profile and escape-analysis
constraints while limiting production claims to what the benchmarks show.
HashtableD2Benchmark no longer says the remaining setup pass was ruled out:
eight passes vs one does not test its effect.

Co-authored-by: bric3 
Co-Authored-By: Claude Opus 5.5 
---
 .../datadog/trace/util/BenchmarkUtils.java    | 68 ++++++-------------
 .../trace/util/HashtableD2Benchmark.java      | 12 ++--
 2 files changed, 28 insertions(+), 52 deletions(-)

diff --git a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
index 3cd5ca58a17..42c294e0469 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/BenchmarkUtils.java
@@ -15,59 +15,35 @@
 import org.openjdk.jmh.infra.Blackhole;
 
 /**
- * Shared setup helpers for JMH benchmarks in this module.
+ * Prepares shared collection code for benchmarks that run in separate JMH forks.
  *
- * 
      These exist to compensate for forking. JMH runs each benchmark method in a fresh JVM, which is
- * right for measurement -- one arm cannot contaminate another's numbers -- but it means every arm
- * is compiled inside a process that has only ever executed that one code path, with that one arm's
- * key types and that one arm's branch outcomes. Production is the reverse: a single JVM runs
- * everything. So the harness hands C2 a maximally specialized view of the world, and the
- * compilation that results can be better than anything achievable in a real application. Put
- * another way, JMH forks to eliminate cross-benchmark profile pollution, but profile pollution
- * is the production condition.
- *
- * 
      That surfaces three ways. This class addresses the first two partially and the third not at
- * all:
+ * 
      Forking isolates benchmark methods, so their receiver types, loaded implementations and branch
+ * outcomes can differ from those in an application. This helper broadens receiver-type profiles and
+ * loads decoy implementations; it does not reproduce a production workload or guarantee a
+ * particular compiled dispatch shape.
  *
  * 
        
- *   
      • Receiver-type profiles collapse toward a single key type, so shared JDK dispatch
- *       sites look monomorphic. {@link #warmUpHashDispatch} drives several key classes through
- *       them.
- *   
      • Class-hierarchy analysis sees only the implementations one arm happens to load, so
- *       C2 can devirtualize calls a real application leaves polymorphic. Loading the decoy
- *       collections widens the hierarchy.
- *   
      • Branch profiles go one-sided, because only one arm's outcomes ever occur. Nothing
- *       here addresses that.
+ *   
      • Receiver-type profiles record the key classes seen at shared collection call sites. {@link
+ *       #warmUpHashDispatch(Blackhole)} exercises those sites with several classes.
+ *   
      • Class-hierarchy analysis uses loaded implementations to resolve virtual calls. Extra decoy
+ *       classes can prevent an assumption that only one implementation exists.
+ *   
      • Branch profiles record which paths execute. A pre-populated, hit-only benchmark may never
+ *       exercise insertion; this helper does not broaden those profiles.
  * 
      
  *
- * 
      {@link Blackhole} is orthogonal to all of this. It stops the JIT proving a result is dead, and
- * says nothing about whether the surrounding code was compiled realistically -- a benchmark can
- * consume every result through a {@code Blackhole} and still measure a devirtualized fast path that
- * cannot occur in production.
- *
- * 
      One-sided branch profiles also interact with escape analysis in a way that can silently
- * flatter a benchmark. When a branch is never taken during profiling, C2 prunes it as an {@code
- * unstable_if} uncommon trap; if that pruned branch held the only store of an object, the object
- * becomes provably non-escaping and escape analysis scalar-replaces an allocation that production
- * would keep. The case measured in this module is {@code map.get(key)} followed by a guarded {@code
- * computeIfAbsent(key, ...)}: with every key pre-installed by {@code @Setup} the absent branch
- * never runs, so a composite key costs nothing at all. A real cache records its population-phase
- * misses in that same branch profile -- MDO counters accumulate from interpretation onward and are
- * never reset -- so the profile is two-sided, nothing is pruned, and the key is allocated on every
- * lookup. See {@code ThreadSafeMapD2Benchmark} for the measurement, and {@code
- * HashtableD2Benchmark} for the contrasting shape, where {@code merge} keeps the present/absent
- * decision inside the callee and leaves no caller-visible branch to prune.
+ * 
      {@link Blackhole} prevents unused results from being eliminated, but does not make the
+ * surrounding type and branch profiles representative of application traffic.
  *
- * 
      The more complete approach is to exercise every benchmark arm during setup, across each of its
- * outcomes, so that each fork's profiles reflect the whole class rather than the one arm it is
- * about to measure. That is what restores the mix the fork removed.
+ * 
      C2 may replace an unobserved miss branch with an uncommon trap, leaving miss handling outside
+ * the optimized path. If that branch was the only path retaining a temporary key, escape analysis
+ * can then eliminate the allocation on hits. {@code ThreadSafeMapD2Benchmark} reports this for a
+ * guarded {@code computeIfAbsent}; {@code HashtableD2Benchmark} reports a retained wrapper for
+ * {@code merge}. Different traffic may change those decisions, so application allocation costs
+ * require representative misses and compiler or allocation profiling.
  *
- * 
      Such a helper should drive the arms in a shuffled order rather than a fixed round-robin, since
- * compilation can trigger part-way through a warmup and would otherwise see whichever arm dominates
- * that point in the sequence; a regular pattern also gives the hardware branch predictor an
- * unrealistically easy time. Seeding the shuffle keeps it irregular but reproducible. A {@code
- * warmUpArms} along those lines is planned as a follow-on, because adopting it changes benchmark
- * setup and requires re-measuring whatever adopts it.
+ * 
      A planned follow-up will exercise each benchmark variant and its relevant outcomes in a
+ * reproducible shuffled order. Compilation can start during setup, so a fixed order may favour
+ * whichever variants run first. Adopting that setup requires remeasuring affected benchmarks.
  */
 public final class BenchmarkUtils {
   private BenchmarkUtils() {}
diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
index 50f8d07c2de..52042fcbce3 100644
--- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
+++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java
@@ -88,12 +88,12 @@
  * is built at the call site with {@code new Key2(...)}, so C2 has an exact type either way and
  * devirtualizes {@code hashCode()}/{@code equals()} without consulting the polluted profile.
  * Pollution therefore cannot explain a move on either side. With the JDK and machine held constant,
- * what remains is uncontrolled run-to-run variation. The other candidate — that {@code
- * warmUpHashDispatch}'s own heavy pre-measurement allocation shifts GC state for the trial — has
- * since been tested and ruled out; see {@link HashtableD1Benchmark} for the 8x-to-1x comparison
- * that moved nothing. Treat these two runs as not directly comparable on absolute numbers. The
- * relative conclusion (D2 dominates {@code update}, leads {@code add} on JDK 8 by avoiding
- * the {@code Key2} allocation, ties on {@code iterate}) is unchanged either way.
+ * what remains is uncontrolled run-to-run variation. The comparison in {@link HashtableD1Benchmark}
+ * reduced setup from eight passes to one without a detectable throughput change. Both runs still
+ * included a pass, so its effect on GC state or throughput remains untested. Treat these two runs
+ * as not directly comparable on absolute numbers. The relative conclusion (D2 dominates
+ * {@code update}, leads {@code add} on JDK 8 by avoiding the {@code Key2} allocation, ties on
+ * {@code iterate}) is unchanged either way.
  *
  * 
      Separately rerun on Zulu 17.0.7 (native AArch64, same machine, pollution wiring unchanged).
  * JMH auto-detected the cheap "compiler" Blackhole mode on Java 17 (its log explicitly warns that