From 90e5a679cb6b7845b0b707f09a6839a7d9e7db25 Mon Sep 17 00:00:00 2001 From: datadog-bits <263423550+datadog-bits@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:49:55 +0000 Subject: [PATCH 1/5] Balance test_inst CI shards by measured duration instead of project-path hash Replace the project-path hash used by `-Pslot=X/Y` with a deterministic greedy longest-processing-time-first bin-packing seeded from per-project test durations harvested from CI Visibility (test-shard-weights.json). Projects missing from the weights file fall back to a default weight. The assignment is computed once per slot count from all project paths, so every shard agrees on it, and cached on the root project. Solves APMCI-45. Co-Authored-By: Claude Opus 5.5 (1M context) Co-authored-by: cgcote <5115638+cgcote@users.noreply.github.com> --- .../gradle/plugin/ci/CIJobsExtensions.kt | 32 ++-- .../gradle/plugin/ci/TestShardBalancer.kt | 68 ++++++++ .../gradle/plugin/ci/test-shard-weights.json | 45 ++++++ .../gradle/plugin/ci/TestShardBalancerTest.kt | 146 ++++++++++++++++++ docs/ci-test-shard-weights.md | 58 +++++++ 5 files changed, 339 insertions(+), 10 deletions(-) create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt create mode 100644 buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json create mode 100644 buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt create mode 100644 docs/ci-test-shard-weights.md diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/CIJobsExtensions.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/CIJobsExtensions.kt index d6269176a0a..5fd6b355bf3 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/CIJobsExtensions.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/CIJobsExtensions.kt @@ -4,7 +4,6 @@ import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.provider.Provider import org.gradle.kotlin.dsl.extra -import kotlin.math.abs /** * Determines if the current project is in the selected slot. @@ -36,15 +35,9 @@ val Project.isInSelectedSlot: Provider return@map true } - // Distribution numbers when running on rootProject.allprojects indicates - // bucket sizes are reasonably balanced: - // - // * size 4 distribution: {2=146, 0=143, 1=157, 3=145} - // * size 6 distribution: {4=100, 0=92, 3=97, 2=97, 1=108, 5=97} - // * size 8 distribution: {2=62, 4=72, 0=71, 5=70, 7=78, 6=84, 1=87, 3=67} - // * size 10 distribution: {8=62, 0=65, 5=70, 9=59, 3=54, 1=56, 6=63, 4=47, 2=52, 7=63} - // * size 12 distribution: {10=55, 0=47, 4=45, 9=46, 8=51, 3=51, 2=46, 1=59, 5=52, 7=49, 11=45, 6=45} - val projectSlot = abs(project.path.hashCode() % totalSlots) + 1 // Convert to 1-based + // Slots are balanced by measured test duration rather than project count, see + // TestShardBalancer and buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json. + val projectSlot = testShardAssignment(totalSlots).getValue(project.path) project.logger.info( "Project {} assigned to slot {}/{}, active slot is {}", @@ -57,6 +50,25 @@ val Project.isInSelectedSlot: Provider projectSlot == selectedSlot }.orElse(true) +/** + * Returns the duration-balanced slot assignment of every project for the given number of slots. + * + * The assignment is computed once per slot count from all project paths, independently of the + * selected slot so that every shard agrees on it, and cached on the root project. + */ +private fun Project.testShardAssignment(totalSlots: Int): Map { + val extra = rootProject.extra + val key = "testShardAssignment.$totalSlots" + synchronized(extra) { + if (!extra.has(key)) { + val projectPaths = rootProject.allprojects.map { it.path }.sorted() + extra.set(key, TestShardBalancer.assignSlots(projectPaths, totalSlots, TestShardBalancer.loadDefaultWeights())) + } + @Suppress("UNCHECKED_CAST") + return extra.get(key) as Map + } +} + /** * Returns the task's path, given affected projects, if this task or its dependencies are affected by git changes. */ diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt new file mode 100644 index 00000000000..1383f419fa2 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt @@ -0,0 +1,68 @@ +package datadog.gradle.plugin.ci + +import com.fasterxml.jackson.databind.ObjectMapper + +/** + * Assigns Gradle projects to CI test shards so that the shards have similar total test duration. + * + * Kept free of Gradle types so the assignment can be unit tested in isolation. + */ +internal object TestShardBalancer { + /** Classpath location of the checked-in per-project test duration weights. */ + const val WEIGHTS_RESOURCE = "/datadog/gradle/plugin/ci/test-shard-weights.json" + + /** Weight, in seconds, of a project that is absent from the weights map. */ + const val DEFAULT_WEIGHT_SECONDS = 18.0 + + /** Parses the `weights` object of a weights file; other top-level keys are ignored. */ + fun loadWeights(json: String): Map { + val weights = ObjectMapper().readTree(json).path("weights") + require(weights.isObject) { "Test shard weights file must contain a 'weights' object" } + val result = LinkedHashMap() + weights.properties().forEach { (path, value) -> + require(value.isNumber && value.asDouble() >= 0.0) { + "Test shard weight for '$path' must be a non-negative number, got '$value'" + } + result[path] = value.asDouble() + } + return result + } + + /** Loads the weights file bundled with buildSrc. */ + fun loadDefaultWeights(): Map { + val stream = checkNotNull(TestShardBalancer::class.java.getResourceAsStream(WEIGHTS_RESOURCE)) { + "Missing test shard weights resource $WEIGHTS_RESOURCE" + } + return loadWeights(stream.bufferedReader().use { it.readText() }) + } + + /** Returns the weight of the given project, falling back to [DEFAULT_WEIGHT_SECONDS]. */ + fun weightOf(projectPath: String, weights: Map): Double = + weights[projectPath] ?: DEFAULT_WEIGHT_SECONDS + + /** + * Assigns every project path to a 1-based slot using greedy longest-processing-time-first + * bin-packing: projects are placed heaviest first (ties broken by path) into the currently + * least loaded slot (ties broken by lowest slot), so the result only depends on the set of + * paths and the weights, never on input order or hash codes. + */ + fun assignSlots(projectPaths: List, totalSlots: Int, weights: Map): Map { + require(totalSlots > 0) { "totalSlots must be positive, got $totalSlots" } + val ordered = projectPaths.distinct().sortedWith( + compareByDescending { weightOf(it, weights) }.thenBy { it } + ) + val loads = DoubleArray(totalSlots) + val assignment = LinkedHashMap() + for (path in ordered) { + var slot = 0 + for (candidate in 1 until totalSlots) { + if (loads[candidate] < loads[slot]) { + slot = candidate + } + } + loads[slot] += weightOf(path, weights) + assignment[path] = slot + 1 + } + return assignment + } +} diff --git a/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json b/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json new file mode 100644 index 00000000000..93f34f640f1 --- /dev/null +++ b/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json @@ -0,0 +1,45 @@ +{ + "_comment": [ + "Per-project test_inst execution weights used to balance CI test shards.", + "Unit: median seconds of test execution time per pipeline run.", + "Source: Datadog CI Visibility, sum(@duration) over test modules grouped by", + "Gradle project, JVM 11, job test_inst, 7-day window ending 2026-10-11,", + "normalized over 491 observed pipelines.", + "Projects absent from this map fall back to the default weight.", + "See docs/ci-test-shard-weights.md to regenerate." + ], + "_metadata": { + "generated": "2026-10-11", + "window_days": 7, + "jvm": "11", + "job": "test_inst", + "pipelines_observed": 491, + "projects_measured": 24 + }, + "weights": { + ":dd-java-agent:instrumentation:jdbc": 1116.7, + ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0": 612.3, + ":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0": 509.4, + ":dd-java-agent:instrumentation:kafka:kafka-clients-0.11": 379.1, + ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.6": 258.0, + ":dd-java-agent:instrumentation:mule-4.5": 217.1, + ":dd-java-agent:instrumentation:grpc-1.5": 196.7, + ":dd-java-agent:instrumentation:maven:maven-3.2.1": 172.2, + ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0": 171.3, + ":dd-java-agent:instrumentation:couchbase:couchbase-3.2": 165.8, + ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7": 159.7, + ":dd-java-agent:instrumentation:couchbase:couchbase-3.1": 151.2, + ":dd-java-agent:instrumentation:vertx:vertx-web:vertx-web-4.0": 140.9, + ":dd-java-agent:instrumentation:spark:spark_2.12": 113.9, + ":dd-java-agent:instrumentation:hibernate:hibernate-core-3.3": 93.6, + ":dd-java-agent:instrumentation:couchbase:couchbase-2.0": 82.5, + ":dd-java-agent:instrumentation:akka:akka-actor-2.5": 66.8, + ":dd-java-agent:instrumentation:jetty:jetty-server:jetty-server-9.4.21": 64.6, + ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.1": 34.7, + ":dd-java-agent:instrumentation:elasticsearch:elasticsearch-rest:elasticsearch-rest-7.0": 31.7, + ":dd-java-agent:instrumentation:couchbase:couchbase-2.6": 25.6, + ":dd-java-agent:instrumentation:spring:spring-webmvc:spring-webmvc-5.3": 21.9, + ":dd-java-agent:instrumentation:spring:spring-boot-1.3": 16.6, + ":dd-java-agent:instrumentation:vertx:vertx-web:vertx-web-3.9": 12.6 + } +} diff --git a/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt b/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt new file mode 100644 index 00000000000..2de3ebcec40 --- /dev/null +++ b/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt @@ -0,0 +1,146 @@ +package datadog.gradle.plugin.ci + +import datadog.gradle.plugin.ci.TestShardBalancer.DEFAULT_WEIGHT_SECONDS +import datadog.gradle.plugin.ci.TestShardBalancer.assignSlots +import datadog.gradle.plugin.ci.TestShardBalancer.loadDefaultWeights +import datadog.gradle.plugin.ci.TestShardBalancer.loadWeights +import datadog.gradle.plugin.ci.TestShardBalancer.weightOf +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import java.io.File +import java.util.TreeSet +import kotlin.random.Random + +class TestShardBalancerTest { + + @ParameterizedTest(name = "{0} slots") + @ValueSource(ints = [1, 2, 4, 6, 8, 12]) + fun `every project is selected by exactly one slot`(totalSlots: Int) { + val assignment = assignSlots(PROJECT_PATHS, totalSlots, WEIGHTS) + + val selectedBySlot = (1..totalSlots).map { slot -> PROJECT_PATHS.filter { assignment[it] == slot } } + assertThat(selectedBySlot.flatten()).containsExactlyInAnyOrderElementsOf(PROJECT_PATHS) + } + + @ParameterizedTest(name = "{0} slots") + @ValueSource(ints = [1, 2, 4, 6, 8, 12]) + fun `assigned slots are within range`(totalSlots: Int) { + val assignment = assignSlots(PROJECT_PATHS, totalSlots, WEIGHTS) + + assertThat(assignment.values).allSatisfy { slot -> assertThat(slot).isBetween(1, totalSlots) } + } + + @Test + fun `assignment does not depend on input order`() { + val expected = assignSlots(PROJECT_PATHS, 8, WEIGHTS) + + assertThat(assignSlots(PROJECT_PATHS, 8, WEIGHTS)).isEqualTo(expected) + for (seed in 1..5) { + assertThat(assignSlots(PROJECT_PATHS.shuffled(Random(seed)), 8, WEIGHTS)).isEqualTo(expected) + } + } + + @Test + fun `instrumentation shards are balanced by duration`() { + val instrumentationPaths = PROJECT_PATHS.filter { it.startsWith(INSTRUMENTATION_PREFIX) } + assertThat(instrumentationPaths).hasSizeGreaterThan(300) + + val assignment = assignSlots(PROJECT_PATHS, 8, WEIGHTS) + + val loads = DoubleArray(8) + instrumentationPaths.forEach { loads[assignment.getValue(it) - 1] += weightOf(it, WEIGHTS) } + assertThat(loads.min()).isPositive() + assertThat(loads.max() / loads.min()).isLessThanOrEqualTo(1.25) + } + + @Test + fun `heaviest projects land in different slots`() { + val assignment = assignSlots(PROJECT_PATHS, 8, WEIGHTS) + + assertThat(assignment[JDBC]).isNotEqualTo(assignment[AKKA_HTTP_10_0]) + val heaviest = WEIGHTS.entries.sortedByDescending { it.value }.take(8).map { it.key } + assertThat(heaviest.map { assignment[it] }).doesNotHaveDuplicates() + } + + @Test + fun `unknown projects receive the default weight`() { + assertThat(weightOf(":unknown", mapOf(":known" to 1.5))).isEqualTo(DEFAULT_WEIGHT_SECONDS) + assertThat(weightOf(":known", mapOf(":known" to 1.5))).isEqualTo(1.5) + + // A known project worth two default weights is balanced by two unknown projects + val assignment = assignSlots( + listOf(":unknown-a", ":known", ":unknown-b"), + 2, + mapOf(":known" to 2 * DEFAULT_WEIGHT_SECONDS) + ) + assertThat(assignment).isEqualTo(mapOf(":known" to 1, ":unknown-a" to 2, ":unknown-b" to 2)) + } + + @Test + fun `single slot selects every project`() { + val assignment = assignSlots(PROJECT_PATHS, 1, WEIGHTS) + + assertThat(assignment).hasSize(PROJECT_PATHS.size) + assertThat(assignment.values).containsOnly(1) + } + + @Test + fun `empty project list yields an empty assignment`() { + assertThat(assignSlots(emptyList(), 8, WEIGHTS)).isEmpty() + } + + @Test + fun `non positive slot count is rejected`() { + assertThatThrownBy { assignSlots(PROJECT_PATHS, 0, WEIGHTS) } + .isInstanceOf(IllegalArgumentException::class.java) + } + + @Test + fun `bundled weights file is parsed`() { + assertThat(WEIGHTS).hasSize(24).containsKeys(JDBC, AKKA_HTTP_10_0) + assertThat(WEIGHTS.values).allSatisfy { weight -> assertThat(weight).isPositive() } + } + + @Test + fun `only the weights object is read`() { + val weights = loadWeights("""{"_comment": ["c"], "_metadata": {"projects_measured": 1}, "weights": {":a": 1.5}}""") + + assertThat(weights).isEqualTo(mapOf(":a" to 1.5)) + } + + @Test + fun `invalid weights are rejected`() { + assertThatThrownBy { loadWeights("""{"_comment": []}""") } + .isInstanceOf(IllegalArgumentException::class.java) + assertThatThrownBy { loadWeights("""{"weights": {":a": -1}}""") } + .isInstanceOf(IllegalArgumentException::class.java) + assertThatThrownBy { loadWeights("""{"weights": {":a": "slow"}}""") } + .isInstanceOf(IllegalArgumentException::class.java) + } + + companion object { + private const val INSTRUMENTATION_PREFIX = ":dd-java-agent:instrumentation:" + private const val JDBC = ":dd-java-agent:instrumentation:jdbc" + private const val AKKA_HTTP_10_0 = ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0" + + private val WEIGHTS = loadDefaultWeights() + + /** Project paths included by the repository settings, plus the implicit parent projects Gradle creates. */ + private val PROJECT_PATHS: List = run { + val settings = File(System.getProperty("user.dir")).parentFile.resolve("settings.gradle.kts") + val includePattern = Regex("\"(:[^\"]+)\"") + val paths = TreeSet(listOf(":")) + settings.readLines() + .filterNot { it.trimStart().startsWith("//") } + .flatMap { line -> includePattern.findAll(line).map { it.groupValues[1] }.toList() } + .forEach { include -> + val segments = include.removePrefix(":").split(":") + segments.indices.forEach { i -> paths.add(":" + segments.subList(0, i + 1).joinToString(":")) } + } + paths.toList() + } + } +} diff --git a/docs/ci-test-shard-weights.md b/docs/ci-test-shard-weights.md new file mode 100644 index 00000000000..05bb86dd55b --- /dev/null +++ b/docs/ci-test-shard-weights.md @@ -0,0 +1,58 @@ +# CI test shard weights + +CI test jobs are split across parallel shards with `-Pslot=X/Y` (for example `test_inst` runs 8 shards). +Each Gradle project is assigned to exactly one shard by +[`TestShardBalancer`](../buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt), which +packs projects by test duration rather than by project count, so that shards finish at about the same time. + +## The weights file + +[`test-shard-weights.json`](../buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json) +maps Gradle project paths to the seconds of test execution each project costs per pipeline run in the `test_inst` +job. Only the `weights` object is read; `_comment` and `_metadata` document where the numbers come from. + +Projects absent from the file get `TestShardBalancer.DEFAULT_WEIGHT_SECONDS` (18 seconds). The file therefore +only needs to list the projects whose tests are much heavier than average. + +Projects are then placed heaviest first (ties broken by project path) into the least loaded shard (ties broken by +lowest shard index). The assignment is computed from all project paths, so it does not depend on which shard is +running, and it does not use hash codes, so it is the same on every JVM and machine. + +## Regenerating the weights + +Weights come from Datadog CI Visibility test events at the **module** level, over a 7-day window: + +``` +@git.repository.id_v2:"github.com/datadog/dd-trace-java" @test.configuration.job_name:test_inst @test.configuration.jvm:11 +``` + +1. Aggregate `sum(@duration)` grouped by `@test.module`. Module names come from the uploaded JUnit reports and look + like `__TEST-.xml`, e.g. `akka-actor-2.5_test_TEST-AkkaActorTest.xml`. +2. Strip the `__TEST-.xml` suffix to get the project name, and sum all modules of a project + (all of its test tasks and test classes). Map the project name to its Gradle project path from + `settings.gradle.kts`, e.g. `jdbc` to `:dd-java-agent:instrumentation:jdbc`. Watch out for project names that + contain underscores, such as `spark_2.12`. +3. Count the observed pipelines with `cardinality(@ci.pipeline.id)` over the same query. +4. Divide each project's total by the pipeline count, and convert from nanoseconds to seconds. + +For example, `jdbc` summed to 548,323.5 seconds over 491 pipelines, giving a weight of `1116.7`. + +Keep the projects that clearly dominate the total, update `_metadata` (`generated`, `pipelines_observed`, +`projects_measured`) and the window end date in `_comment`, update the expected entry count in +`TestShardBalancerTest`, then run the buildSrc tests: + +```shell +./gradlew :buildSrc:test -PrunBuildSrcTests +``` + +`TestShardBalancerTest` checks that the instrumentation shards stay balanced, with a max/min load ratio of at most +1.25. + +## When to refresh + +- After adding, splitting, or renaming a heavy instrumentation module, or one that adds many slow tests. +- When CI Visibility shows the `test_inst` shard durations drifting apart again. + +Stale weights only make the balance worse. They never drop test coverage: every project is always assigned to exactly +one shard, and projects missing from the file (including new or renamed ones) fall back to the default weight. +Weights for projects that no longer exist are ignored. From ba1be0dc5420a72eba3f2f8e8fa3404b5ae5f595 Mon Sep 17 00:00:00 2001 From: datadog-bits <263423550+datadog-bits@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:00:27 +0000 Subject: [PATCH 2/5] Cache muzzle classloaders per bootstrap class-path MuzzleAction cached a single boot/tool classloader pair per build and only rebuilt it when the bootstrap + tooling class-path file count grew. A project whose muzzleBootstrap differs from, but is not larger than, the previous project's therefore reused the wrong bootstrap classloader: - graal-native-image-20.0 adds agent-installer to muzzleBootstrap; - opentelemetry-1.47 adds otel-bootstrap, with fewer files; - when graal ran first in the same worker, opentelemetry-1.47 reused its classloader and failed helper injection with NoClassDefFoundError: datadog/trace/bootstrap/otel/metrics/OtelInstrument. The reverse also held: projects with a smaller bootstrap class-path could see classes they never declared. Under project-path hashing, graal and the OpenTelemetry projects never shared a muzzle shard; duration-based shard assignment put graal-native-image-20.0 and opentelemetry-1.47 together in slot 5, exposing it. Cache the classloaders per distinct class-path for the current build so every muzzle check sees exactly its own declared bootstrap class-path. Co-Authored-By: Claude Opus 5.5 (1M context) Co-authored-by: cgcote <5115638+cgcote@users.noreply.github.com> --- .../gradle/plugin/muzzle/MuzzleAction.kt | 64 ++++++++++--------- .../gradle/plugin/muzzle/MuzzleActionTest.kt | 57 +++++++++++++++++ 2 files changed, 90 insertions(+), 31 deletions(-) create mode 100644 buildSrc/src/test/kotlin/datadog/gradle/plugin/muzzle/MuzzleActionTest.kt diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzleAction.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzleAction.kt index aa7309a534b..6ec14527562 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzleAction.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzleAction.kt @@ -1,58 +1,60 @@ package datadog.gradle.plugin.muzzle import org.gradle.api.GradleException -import org.gradle.api.file.FileCollection import org.gradle.workers.WorkAction +import java.io.File import java.lang.reflect.Method import java.net.URLClassLoader abstract class MuzzleAction : WorkAction { companion object { private val lock = Any() - @Volatile - private var lastBootCL: ClassLoader? = null - @Volatile - private var lastToolCL: ClassLoader? = null - @Volatile - private var lastBuildStamp: Long = 0 - @Volatile - private var lastBuildPathCount: Int = 0 + private var cachedBuildStamp: Long = 0 + private val cachedClassLoaders = HashMap, List>, Pair>() /** * Creates a classloader for [cp] with the given [parent]. By default, the platform * classloader keeps JDK classes visible without exposing Gradle's bundled libraries. */ fun createClassLoader( - cp: FileCollection, + cp: Iterable, parent: ClassLoader = ClassLoader.getPlatformClassLoader() ): ClassLoader { val urls = cp.map { it.toURI().toURL() }.toTypedArray() return URLClassLoader(urls, parent) } - } - override fun execute() { - val buildStamp = parameters.buildStartedTime.get() - val buildPathCount = parameters.bootstrapClassPath.count() + parameters.toolingClassPath.count() - var bootCL : ClassLoader? = lastBootCL - var toolCL : ClassLoader? = lastToolCL - // cache boot and tool classloaders for each run; rebuild if either class-path is extended mid-build - if (bootCL == null || toolCL == null || lastBuildStamp < buildStamp || lastBuildPathCount < buildPathCount) { - synchronized(lock) { - bootCL = lastBootCL - toolCL = lastToolCL - if (bootCL == null || toolCL == null || lastBuildStamp < buildStamp || lastBuildPathCount < buildPathCount) { - bootCL = createClassLoader(parameters.bootstrapClassPath) - toolCL = createClassLoader(parameters.toolingClassPath, bootCL!!) - lastBootCL = bootCL - lastToolCL = toolCL - lastBuildStamp = buildStamp - lastBuildPathCount = buildPathCount - } + /** + * Returns the boot and tool classloaders for the given class-paths, cached for the current build. + * + * Projects can extend the bootstrap class-path (see `muzzleBootstrap`), so classloaders are cached + * per distinct class-path: reusing the ones built for another project would hide classes this + * project declares, or expose classes it does not declare. + */ + internal fun bootAndToolClassLoaders( + buildStamp: Long, + bootstrapClassPath: List, + toolingClassPath: List + ): Pair = synchronized(lock) { + if (cachedBuildStamp < buildStamp) { + cachedClassLoaders.clear() + cachedBuildStamp = buildStamp + } + cachedClassLoaders.getOrPut(bootstrapClassPath to toolingClassPath) { + val bootCL = createClassLoader(bootstrapClassPath) + bootCL to createClassLoader(toolingClassPath, bootCL) } } - val instCL = createClassLoader(parameters.instrumentationClassPath, toolCL!!) - val testCL = createClassLoader(parameters.testApplicationClassPath, bootCL!!) + } + + override fun execute() { + val (bootCL, toolCL) = bootAndToolClassLoaders( + parameters.buildStartedTime.get(), + parameters.bootstrapClassPath.files.toList(), + parameters.toolingClassPath.files.toList() + ) + val instCL = createClassLoader(parameters.instrumentationClassPath, toolCL) + val testCL = createClassLoader(parameters.testApplicationClassPath, bootCL) val assertPass = parameters.assertPass.get() val muzzleDirective = parameters.muzzleDirective.orNull val assertionMethod: Method = instCL.loadClass("datadog.trace.agent.tooling.muzzle.MuzzleVersionScanPlugin") diff --git a/buildSrc/src/test/kotlin/datadog/gradle/plugin/muzzle/MuzzleActionTest.kt b/buildSrc/src/test/kotlin/datadog/gradle/plugin/muzzle/MuzzleActionTest.kt new file mode 100644 index 00000000000..8e05501d619 --- /dev/null +++ b/buildSrc/src/test/kotlin/datadog/gradle/plugin/muzzle/MuzzleActionTest.kt @@ -0,0 +1,57 @@ +package datadog.gradle.plugin.muzzle + +import datadog.gradle.plugin.muzzle.MuzzleAction.Companion.bootAndToolClassLoaders +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File +import java.util.concurrent.atomic.AtomicLong + +class MuzzleActionTest { + + @TempDir + lateinit var tempDir: File + + @Test + fun `bootstrap classloader only exposes the requested class-path`() { + val base = classPathEntry("base") + val otel = classPathEntry("otel") + val tooling = listOf(classPathEntry("tooling")) + val buildStamp = nextBuildStamp() + + // A larger, unrelated bootstrap class-path checked first in the same build, like + // graal-native-image-20.0 adding agent-installer before opentelemetry-1.47 adds otel-bootstrap + bootAndToolClassLoaders(buildStamp, listOf(base, classPathEntry("installer"), classPathEntry("installer-deps")), tooling) + val (bootCL, toolCL) = bootAndToolClassLoaders(buildStamp, listOf(base, otel), tooling) + + assertThat(bootCL.getResource("otel.marker")).isNotNull() + assertThat(bootCL.getResource("installer.marker")).isNull() + assertThat(toolCL.getResource("otel.marker")).isNotNull() + assertThat(toolCL.getResource("tooling.marker")).isNotNull() + } + + @Test + fun `classloaders are reused for the same class-path within a build`() { + val bootstrap = listOf(classPathEntry("base")) + val tooling = listOf(classPathEntry("tooling")) + val buildStamp = nextBuildStamp() + + val classLoaders = bootAndToolClassLoaders(buildStamp, bootstrap, tooling) + + assertThat(bootAndToolClassLoaders(buildStamp, bootstrap, tooling)).isSameAs(classLoaders) + assertThat(bootAndToolClassLoaders(nextBuildStamp(), bootstrap, tooling)).isNotSameAs(classLoaders) + } + + /** Creates a class-path directory holding a `.marker` resource. */ + private fun classPathEntry(name: String): File = + File(tempDir, name).apply { + mkdirs() + File(this, "$name.marker").writeText(name) + } + + companion object { + private val buildStamps = AtomicLong(System.currentTimeMillis()) + + private fun nextBuildStamp() = buildStamps.incrementAndGet() + } +} From 782644ad8d0af8bdf568ba2bf474ecde418ddcb9 Mon Sep 17 00:00:00 2001 From: datadog-bits <263423550+datadog-bits@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:33:22 +0000 Subject: [PATCH 3/5] retrigger CI Co-authored-by: cgcote <5115638+cgcote@users.noreply.github.com> From 00bf834ff690d4233a7516ad92b6a844b9cefe7f Mon Sep 17 00:00:00 2001 From: datadog-bits <263423550+datadog-bits@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:55:37 +0000 Subject: [PATCH 4/5] Re-harvest test_inst shard weights and document the single-project floor The first pipeline using duration-based shards did not speed up the slowest test_inst shard: the 24-entry weights file missed lettuce-5.0, the second heaviest project, which then shared slot 3 with spring-webflux-5.0. - Replace the weights with 36 projects measured one by one from CI Visibility (same query, JVM 11, 7 days, 491 pipelines), and correct the unit to a per-pipeline mean. - Raise DEFAULT_WEIGHT_SECONDS to 20.6, the mean of the projects left out. - Test that the two heaviest projects never share a slot and that every weighted path is an included project. - Document that a shard can never finish faster than its heaviest project (jdbc, then lettuce-5.0), so the modeled balance is optimistic when one project approaches the target shard time. Co-Authored-By: Claude Opus 5.5 (1M context) Co-authored-by: cgcote <5115638+cgcote@users.noreply.github.com> --- .../gradle/plugin/ci/TestShardBalancer.kt | 2 +- .../gradle/plugin/ci/test-shard-weights.json | 20 +++++-- .../gradle/plugin/ci/TestShardBalancerTest.kt | 47 +++++++++++----- docs/ci-test-shard-weights.md | 54 +++++++++++++++---- 4 files changed, 97 insertions(+), 26 deletions(-) diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt index 1383f419fa2..813c3ffd0d4 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt @@ -12,7 +12,7 @@ internal object TestShardBalancer { const val WEIGHTS_RESOURCE = "/datadog/gradle/plugin/ci/test-shard-weights.json" /** Weight, in seconds, of a project that is absent from the weights map. */ - const val DEFAULT_WEIGHT_SECONDS = 18.0 + const val DEFAULT_WEIGHT_SECONDS = 20.6 /** Parses the `weights` object of a weights file; other top-level keys are ignored. */ fun loadWeights(json: String): Map { diff --git a/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json b/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json index 93f34f640f1..4b1c38c2305 100644 --- a/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json +++ b/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json @@ -1,43 +1,55 @@ { "_comment": [ "Per-project test_inst execution weights used to balance CI test shards.", - "Unit: median seconds of test execution time per pipeline run.", + "Unit: mean seconds of test execution time per pipeline run.", "Source: Datadog CI Visibility, sum(@duration) over test modules grouped by", "Gradle project, JVM 11, job test_inst, 7-day window ending 2026-10-11,", - "normalized over 491 observed pipelines.", + "divided by the 491 observed pipelines.", "Projects absent from this map fall back to the default weight.", "See docs/ci-test-shard-weights.md to regenerate." ], "_metadata": { - "generated": "2026-10-11", + "generated": "2026-10-12", "window_days": 7, "jvm": "11", "job": "test_inst", "pipelines_observed": 491, - "projects_measured": 24 + "projects_measured": 36 }, "weights": { ":dd-java-agent:instrumentation:jdbc": 1116.7, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0": 978.1, ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0": 612.3, ":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0": 509.4, ":dd-java-agent:instrumentation:kafka:kafka-clients-0.11": 379.1, ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.6": 258.0, + ":dd-java-agent:instrumentation:aws-java:aws-java-sdk-2.2": 220.7, ":dd-java-agent:instrumentation:mule-4.5": 217.1, ":dd-java-agent:instrumentation:grpc-1.5": 196.7, + ":dd-java-agent:instrumentation:junit:junit-5:junit-5.3": 195.4, + ":dd-java-agent:instrumentation:liberty:liberty-20.0": 187.3, + ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.8": 175.4, ":dd-java-agent:instrumentation:maven:maven-3.2.1": 172.2, ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0": 171.3, ":dd-java-agent:instrumentation:couchbase:couchbase-3.2": 165.8, ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7": 159.7, + ":dd-java-agent:instrumentation:aws-java:aws-java-sqs-1.0": 152.1, ":dd-java-agent:instrumentation:couchbase:couchbase-3.1": 151.2, + ":dd-java-agent:instrumentation:aws-java:aws-java-sqs-2.0": 148.7, ":dd-java-agent:instrumentation:vertx:vertx-web:vertx-web-4.0": 140.9, ":dd-java-agent:instrumentation:spark:spark_2.12": 113.9, + ":dd-java-agent:instrumentation:aws-java:aws-java-sdk-1.11": 104.0, ":dd-java-agent:instrumentation:hibernate:hibernate-core-3.3": 93.6, + ":dd-java-agent:instrumentation:jersey:jersey-2.0": 84.8, ":dd-java-agent:instrumentation:couchbase:couchbase-2.0": 82.5, ":dd-java-agent:instrumentation:akka:akka-actor-2.5": 66.8, ":dd-java-agent:instrumentation:jetty:jetty-server:jetty-server-9.4.21": 64.6, + ":dd-java-agent:instrumentation:lettuce:lettuce-4.0": 47.3, + ":dd-java-agent:instrumentation:jedis:jedis-3.0": 42.0, ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.1": 34.7, ":dd-java-agent:instrumentation:elasticsearch:elasticsearch-rest:elasticsearch-rest-7.0": 31.7, ":dd-java-agent:instrumentation:couchbase:couchbase-2.6": 25.6, + ":dd-java-agent:instrumentation:synapse-3.0": 24.5, ":dd-java-agent:instrumentation:spring:spring-webmvc:spring-webmvc-5.3": 21.9, ":dd-java-agent:instrumentation:spring:spring-boot-1.3": 16.6, ":dd-java-agent:instrumentation:vertx:vertx-web:vertx-web-3.9": 12.6 diff --git a/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt b/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt index 2de3ebcec40..9765a436492 100644 --- a/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt +++ b/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt @@ -56,13 +56,22 @@ class TestShardBalancerTest { assertThat(loads.max() / loads.min()).isLessThanOrEqualTo(1.25) } + @ParameterizedTest(name = "{0} slots") + @ValueSource(ints = [2, 4, 6, 8, 12]) + fun `two heaviest projects never share a slot`(totalSlots: Int) { + // Each one sets a floor on its shard's duration, so sharing a slot would stack the floors + val (heaviest, secondHeaviest) = heaviestProjects(2) + + val assignment = assignSlots(PROJECT_PATHS, totalSlots, WEIGHTS) + + assertThat(assignment.getValue(heaviest)).isNotEqualTo(assignment.getValue(secondHeaviest)) + } + @Test fun `heaviest projects land in different slots`() { val assignment = assignSlots(PROJECT_PATHS, 8, WEIGHTS) - assertThat(assignment[JDBC]).isNotEqualTo(assignment[AKKA_HTTP_10_0]) - val heaviest = WEIGHTS.entries.sortedByDescending { it.value }.take(8).map { it.key } - assertThat(heaviest.map { assignment[it] }).doesNotHaveDuplicates() + assertThat(heaviestProjects(8).map { assignment.getValue(it) }).doesNotHaveDuplicates() } @Test @@ -100,8 +109,15 @@ class TestShardBalancerTest { @Test fun `bundled weights file is parsed`() { - assertThat(WEIGHTS).hasSize(24).containsKeys(JDBC, AKKA_HTTP_10_0) + assertThat(WEIGHTS).hasSize(36) assertThat(WEIGHTS.values).allSatisfy { weight -> assertThat(weight).isPositive() } + // docs/ci-test-shard-weights.md names the two heaviest projects + assertThat(heaviestProjects(2)).containsExactly(JDBC, LETTUCE_5_0) + } + + @Test + fun `every weighted path is an included project`() { + assertThat(INCLUDED_PATHS).containsAll(WEIGHTS.keys) } @Test @@ -124,23 +140,30 @@ class TestShardBalancerTest { companion object { private const val INSTRUMENTATION_PREFIX = ":dd-java-agent:instrumentation:" private const val JDBC = ":dd-java-agent:instrumentation:jdbc" - private const val AKKA_HTTP_10_0 = ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0" + private const val LETTUCE_5_0 = ":dd-java-agent:instrumentation:lettuce:lettuce-5.0" private val WEIGHTS = loadDefaultWeights() - /** Project paths included by the repository settings, plus the implicit parent projects Gradle creates. */ - private val PROJECT_PATHS: List = run { + /** Project paths included by the repository settings. */ + private val INCLUDED_PATHS: List = run { val settings = File(System.getProperty("user.dir")).parentFile.resolve("settings.gradle.kts") val includePattern = Regex("\"(:[^\"]+)\"") - val paths = TreeSet(listOf(":")) settings.readLines() .filterNot { it.trimStart().startsWith("//") } .flatMap { line -> includePattern.findAll(line).map { it.groupValues[1] }.toList() } - .forEach { include -> - val segments = include.removePrefix(":").split(":") - segments.indices.forEach { i -> paths.add(":" + segments.subList(0, i + 1).joinToString(":")) } - } + } + + /** Included project paths, plus the implicit parent projects Gradle creates. */ + private val PROJECT_PATHS: List = run { + val paths = TreeSet(listOf(":")) + INCLUDED_PATHS.forEach { include -> + val segments = include.removePrefix(":").split(":") + segments.indices.forEach { i -> paths.add(":" + segments.subList(0, i + 1).joinToString(":")) } + } paths.toList() } + + private fun heaviestProjects(count: Int): List = + WEIGHTS.entries.sortedByDescending { it.value }.take(count).map { it.key } } } diff --git a/docs/ci-test-shard-weights.md b/docs/ci-test-shard-weights.md index 05bb86dd55b..b805040c554 100644 --- a/docs/ci-test-shard-weights.md +++ b/docs/ci-test-shard-weights.md @@ -11,8 +11,10 @@ packs projects by test duration rather than by project count, so that shards fin maps Gradle project paths to the seconds of test execution each project costs per pipeline run in the `test_inst` job. Only the `weights` object is read; `_comment` and `_metadata` document where the numbers come from. -Projects absent from the file get `TestShardBalancer.DEFAULT_WEIGHT_SECONDS` (18 seconds). The file therefore -only needs to list the projects whose tests are much heavier than average. +Projects absent from the file get `TestShardBalancer.DEFAULT_WEIGHT_SECONDS` (20.6 seconds, the mean weight of the +measured projects that are left out of the file). The file therefore only needs to list the projects whose tests are +much heavier than average, but it must not miss any of the heaviest ones: a heavy project that falls back to the +default weight can overload whichever shard it lands in. Projects are then placed heaviest first (ties broken by project path) into the least loaded shard (ties broken by lowest shard index). The assignment is computed from all project paths, so it does not depend on which shard is @@ -33,20 +35,53 @@ Weights come from Datadog CI Visibility test events at the **module** level, ove `settings.gradle.kts`, e.g. `jdbc` to `:dd-java-agent:instrumentation:jdbc`. Watch out for project names that contain underscores, such as `spark_2.12`. 3. Count the observed pipelines with `cardinality(@ci.pipeline.id)` over the same query. -4. Divide each project's total by the pipeline count, and convert from nanoseconds to seconds. +4. Divide each project's total by the pipeline count, and convert from nanoseconds to seconds. The result is a mean + per pipeline, not a median. For example, `jdbc` summed to 548,323.5 seconds over 491 pipelines, giving a weight of `1116.7`. -Keep the projects that clearly dominate the total, update `_metadata` (`generated`, `pipelines_observed`, -`projects_measured`) and the window end date in `_comment`, update the expected entry count in -`TestShardBalancerTest`, then run the buildSrc tests: +Measure every project listed in `settings.gradle.kts` one by one (e.g. `@test.module:lettuce-5.0_*`) instead of +reading a single grouped aggregation or guessing family prefixes: + +- grouped aggregations can silently return only the first few buckets; +- a prefix that matches nothing (e.g. `aws-sdk*` when the projects are named `aws-java-sdk-*`) means the prefix is + wrong, not that the project is light. + +Keep the projects that clearly dominate the total, and set `DEFAULT_WEIGHT_SECONDS` to the mean weight of the +projects left out. Update `_metadata` (`generated`, `pipelines_observed`, `projects_measured`) and the window end +date in `_comment`, update the expected entry count in `TestShardBalancerTest`, then run the buildSrc tests: ```shell ./gradlew :buildSrc:test -PrunBuildSrcTests ``` -`TestShardBalancerTest` checks that the instrumentation shards stay balanced, with a max/min load ratio of at most -1.25. +`TestShardBalancerTest` checks that every weighted path is a project from `settings.gradle.kts`, that the two +heaviest projects land in different shards, and that the modeled instrumentation shards stay balanced, with a +max/min load ratio of at most 1.25. That ratio is computed from the weights, not measured, see below. + +## Known limitation: the heaviest project sets a floor + +A project's tests always run in a single shard, and tests run in parallel within a shard. The balancer only adds up +seconds, so it cannot see that a shard dominated by one large project finishes later than a shard with the same total +spread over many small projects. In the first pipeline using this balancer, shards 1/8 and 4/8 ran almost the same +total test time (2312 s and 2292 s), yet took 39 and 24 minutes: shard 1/8 held `jdbc`. + +The two heaviest projects per pipeline are currently: + +- `:dd-java-agent:instrumentation:jdbc`: 1116.7 s, about 18.6 minutes; +- `:dd-java-agent:instrumentation:lettuce:lettuce-5.0`: 978.1 s, about 16.3 minutes. + +The slowest shard can therefore never be faster than the heaviest single project, plus job setup. A project's +wall-clock span is also longer than its summed test time: in that first pipeline, the `jdbc` tests (20.5 minutes of +summed test time) spanned 27.6 minutes, starting about 10 minutes into the job, so its shard took 39 minutes. Under +the previous hash-based assignment, the `jdbc` shard was already the slowest one. + +The modeled max/min ratio is optimistic whenever one project approaches the target shard time, so treat the 1.25 +bound as a check of the model rather than a guarantee for wall-clock time. + +The two heaviest projects must not share a shard, otherwise their floors stack. Capping weights would not help: +under-weighting `jdbc` makes the model look balanced while letting more work pile onto the `jdbc` shard. Lowering the +floor needs finer-grained sharding than whole projects. ## When to refresh @@ -55,4 +90,5 @@ Keep the projects that clearly dominate the total, update `_metadata` (`generate Stale weights only make the balance worse. They never drop test coverage: every project is always assigned to exactly one shard, and projects missing from the file (including new or renamed ones) fall back to the default weight. -Weights for projects that no longer exist are ignored. +After renaming or removing a weighted project, update its entry: `TestShardBalancerTest` fails on weights for paths +that are not projects. From 6212592979f107a2ac7df08f79c3fef7272154ec Mon Sep 17 00:00:00 2001 From: datadog-bits <263423550+datadog-bits@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:55:37 +0000 Subject: [PATCH 5/5] Declare otel-bootstrap on muzzleBootstrap for opentelemetry-annotations-1.20 otel-shim, an implementation dependency of this module, is compiled against otel-bootstrap classes that the agent loads on the bootstrap class-path. Declare them for muzzle like the sibling opentelemetry modules do, so muzzle checks this module against the same bootstrap classes. This is a consistency improvement only: muzzle passes for this module with and without the declaration, and it was not the cause of the muzzle: [5/8] failure, which came from the muzzle classloader cache. Co-Authored-By: Claude Opus 5.5 (1M context) Co-authored-by: cgcote <5115638+cgcote@users.noreply.github.com> --- .../opentelemetry/opentelemetry-annotations-1.20/build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-annotations-1.20/build.gradle b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-annotations-1.20/build.gradle index 3811b255e1e..55b3bf4fddb 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-annotations-1.20/build.gradle +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-annotations-1.20/build.gradle @@ -21,6 +21,10 @@ dependencies { compileOnly group: 'com.google.auto.value', name: 'auto-value-annotations', version: '1.6.6' implementation project(':dd-java-agent:agent-otel:otel-shim') + + // otel-shim is compiled against otel-bootstrap classes (e.g. OtelInstrument) that the agent loads on the bootstrap + // class-path, so muzzle checks this module against the same bootstrap classes, like the sibling opentelemetry modules. + muzzleBootstrap project(path: ':dd-java-agent:agent-otel:otel-bootstrap', configuration: 'shadow') testImplementation group: 'io.opentelemetry', name: 'opentelemetry-api', version: openTelemetryVersion testImplementation group: 'io.opentelemetry.instrumentation', name: 'opentelemetry-instrumentation-annotations', version: openTelemetryVersion latest1xDepTestImplementation group: 'io.opentelemetry.instrumentation', name: 'opentelemetry-instrumentation-annotations', version: '1+'