diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/CIJobsExtensions.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/CIJobsExtensions.kt index d6269176a0a..5fd6b355bf3 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/CIJobsExtensions.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/CIJobsExtensions.kt @@ -4,7 +4,6 @@ import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.provider.Provider import org.gradle.kotlin.dsl.extra -import kotlin.math.abs /** * Determines if the current project is in the selected slot. @@ -36,15 +35,9 @@ val Project.isInSelectedSlot: Provider return@map true } - // Distribution numbers when running on rootProject.allprojects indicates - // bucket sizes are reasonably balanced: - // - // * size 4 distribution: {2=146, 0=143, 1=157, 3=145} - // * size 6 distribution: {4=100, 0=92, 3=97, 2=97, 1=108, 5=97} - // * size 8 distribution: {2=62, 4=72, 0=71, 5=70, 7=78, 6=84, 1=87, 3=67} - // * size 10 distribution: {8=62, 0=65, 5=70, 9=59, 3=54, 1=56, 6=63, 4=47, 2=52, 7=63} - // * size 12 distribution: {10=55, 0=47, 4=45, 9=46, 8=51, 3=51, 2=46, 1=59, 5=52, 7=49, 11=45, 6=45} - val projectSlot = abs(project.path.hashCode() % totalSlots) + 1 // Convert to 1-based + // Slots are balanced by measured test duration rather than project count, see + // TestShardBalancer and buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json. + val projectSlot = testShardAssignment(totalSlots).getValue(project.path) project.logger.info( "Project {} assigned to slot {}/{}, active slot is {}", @@ -57,6 +50,25 @@ val Project.isInSelectedSlot: Provider projectSlot == selectedSlot }.orElse(true) +/** + * Returns the duration-balanced slot assignment of every project for the given number of slots. + * + * The assignment is computed once per slot count from all project paths, independently of the + * selected slot so that every shard agrees on it, and cached on the root project. + */ +private fun Project.testShardAssignment(totalSlots: Int): Map { + val extra = rootProject.extra + val key = "testShardAssignment.$totalSlots" + synchronized(extra) { + if (!extra.has(key)) { + val projectPaths = rootProject.allprojects.map { it.path }.sorted() + extra.set(key, TestShardBalancer.assignSlots(projectPaths, totalSlots, TestShardBalancer.loadDefaultWeights())) + } + @Suppress("UNCHECKED_CAST") + return extra.get(key) as Map + } +} + /** * Returns the task's path, given affected projects, if this task or its dependencies are affected by git changes. */ diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt new file mode 100644 index 00000000000..813c3ffd0d4 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt @@ -0,0 +1,68 @@ +package datadog.gradle.plugin.ci + +import com.fasterxml.jackson.databind.ObjectMapper + +/** + * Assigns Gradle projects to CI test shards so that the shards have similar total test duration. + * + * Kept free of Gradle types so the assignment can be unit tested in isolation. + */ +internal object TestShardBalancer { + /** Classpath location of the checked-in per-project test duration weights. */ + const val WEIGHTS_RESOURCE = "/datadog/gradle/plugin/ci/test-shard-weights.json" + + /** Weight, in seconds, of a project that is absent from the weights map. */ + const val DEFAULT_WEIGHT_SECONDS = 20.6 + + /** Parses the `weights` object of a weights file; other top-level keys are ignored. */ + fun loadWeights(json: String): Map { + val weights = ObjectMapper().readTree(json).path("weights") + require(weights.isObject) { "Test shard weights file must contain a 'weights' object" } + val result = LinkedHashMap() + weights.properties().forEach { (path, value) -> + require(value.isNumber && value.asDouble() >= 0.0) { + "Test shard weight for '$path' must be a non-negative number, got '$value'" + } + result[path] = value.asDouble() + } + return result + } + + /** Loads the weights file bundled with buildSrc. */ + fun loadDefaultWeights(): Map { + val stream = checkNotNull(TestShardBalancer::class.java.getResourceAsStream(WEIGHTS_RESOURCE)) { + "Missing test shard weights resource $WEIGHTS_RESOURCE" + } + return loadWeights(stream.bufferedReader().use { it.readText() }) + } + + /** Returns the weight of the given project, falling back to [DEFAULT_WEIGHT_SECONDS]. */ + fun weightOf(projectPath: String, weights: Map): Double = + weights[projectPath] ?: DEFAULT_WEIGHT_SECONDS + + /** + * Assigns every project path to a 1-based slot using greedy longest-processing-time-first + * bin-packing: projects are placed heaviest first (ties broken by path) into the currently + * least loaded slot (ties broken by lowest slot), so the result only depends on the set of + * paths and the weights, never on input order or hash codes. + */ + fun assignSlots(projectPaths: List, totalSlots: Int, weights: Map): Map { + require(totalSlots > 0) { "totalSlots must be positive, got $totalSlots" } + val ordered = projectPaths.distinct().sortedWith( + compareByDescending { weightOf(it, weights) }.thenBy { it } + ) + val loads = DoubleArray(totalSlots) + val assignment = LinkedHashMap() + for (path in ordered) { + var slot = 0 + for (candidate in 1 until totalSlots) { + if (loads[candidate] < loads[slot]) { + slot = candidate + } + } + loads[slot] += weightOf(path, weights) + assignment[path] = slot + 1 + } + return assignment + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzleAction.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzleAction.kt index aa7309a534b..6ec14527562 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzleAction.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzleAction.kt @@ -1,58 +1,60 @@ package datadog.gradle.plugin.muzzle import org.gradle.api.GradleException -import org.gradle.api.file.FileCollection import org.gradle.workers.WorkAction +import java.io.File import java.lang.reflect.Method import java.net.URLClassLoader abstract class MuzzleAction : WorkAction { companion object { private val lock = Any() - @Volatile - private var lastBootCL: ClassLoader? = null - @Volatile - private var lastToolCL: ClassLoader? = null - @Volatile - private var lastBuildStamp: Long = 0 - @Volatile - private var lastBuildPathCount: Int = 0 + private var cachedBuildStamp: Long = 0 + private val cachedClassLoaders = HashMap, List>, Pair>() /** * Creates a classloader for [cp] with the given [parent]. By default, the platform * classloader keeps JDK classes visible without exposing Gradle's bundled libraries. */ fun createClassLoader( - cp: FileCollection, + cp: Iterable, parent: ClassLoader = ClassLoader.getPlatformClassLoader() ): ClassLoader { val urls = cp.map { it.toURI().toURL() }.toTypedArray() return URLClassLoader(urls, parent) } - } - override fun execute() { - val buildStamp = parameters.buildStartedTime.get() - val buildPathCount = parameters.bootstrapClassPath.count() + parameters.toolingClassPath.count() - var bootCL : ClassLoader? = lastBootCL - var toolCL : ClassLoader? = lastToolCL - // cache boot and tool classloaders for each run; rebuild if either class-path is extended mid-build - if (bootCL == null || toolCL == null || lastBuildStamp < buildStamp || lastBuildPathCount < buildPathCount) { - synchronized(lock) { - bootCL = lastBootCL - toolCL = lastToolCL - if (bootCL == null || toolCL == null || lastBuildStamp < buildStamp || lastBuildPathCount < buildPathCount) { - bootCL = createClassLoader(parameters.bootstrapClassPath) - toolCL = createClassLoader(parameters.toolingClassPath, bootCL!!) - lastBootCL = bootCL - lastToolCL = toolCL - lastBuildStamp = buildStamp - lastBuildPathCount = buildPathCount - } + /** + * Returns the boot and tool classloaders for the given class-paths, cached for the current build. + * + * Projects can extend the bootstrap class-path (see `muzzleBootstrap`), so classloaders are cached + * per distinct class-path: reusing the ones built for another project would hide classes this + * project declares, or expose classes it does not declare. + */ + internal fun bootAndToolClassLoaders( + buildStamp: Long, + bootstrapClassPath: List, + toolingClassPath: List + ): Pair = synchronized(lock) { + if (cachedBuildStamp < buildStamp) { + cachedClassLoaders.clear() + cachedBuildStamp = buildStamp + } + cachedClassLoaders.getOrPut(bootstrapClassPath to toolingClassPath) { + val bootCL = createClassLoader(bootstrapClassPath) + bootCL to createClassLoader(toolingClassPath, bootCL) } } - val instCL = createClassLoader(parameters.instrumentationClassPath, toolCL!!) - val testCL = createClassLoader(parameters.testApplicationClassPath, bootCL!!) + } + + override fun execute() { + val (bootCL, toolCL) = bootAndToolClassLoaders( + parameters.buildStartedTime.get(), + parameters.bootstrapClassPath.files.toList(), + parameters.toolingClassPath.files.toList() + ) + val instCL = createClassLoader(parameters.instrumentationClassPath, toolCL) + val testCL = createClassLoader(parameters.testApplicationClassPath, bootCL) val assertPass = parameters.assertPass.get() val muzzleDirective = parameters.muzzleDirective.orNull val assertionMethod: Method = instCL.loadClass("datadog.trace.agent.tooling.muzzle.MuzzleVersionScanPlugin") diff --git a/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json b/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json new file mode 100644 index 00000000000..4b1c38c2305 --- /dev/null +++ b/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json @@ -0,0 +1,57 @@ +{ + "_comment": [ + "Per-project test_inst execution weights used to balance CI test shards.", + "Unit: mean seconds of test execution time per pipeline run.", + "Source: Datadog CI Visibility, sum(@duration) over test modules grouped by", + "Gradle project, JVM 11, job test_inst, 7-day window ending 2026-10-11,", + "divided by the 491 observed pipelines.", + "Projects absent from this map fall back to the default weight.", + "See docs/ci-test-shard-weights.md to regenerate." + ], + "_metadata": { + "generated": "2026-10-12", + "window_days": 7, + "jvm": "11", + "job": "test_inst", + "pipelines_observed": 491, + "projects_measured": 36 + }, + "weights": { + ":dd-java-agent:instrumentation:jdbc": 1116.7, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0": 978.1, + ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0": 612.3, + ":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0": 509.4, + ":dd-java-agent:instrumentation:kafka:kafka-clients-0.11": 379.1, + ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.6": 258.0, + ":dd-java-agent:instrumentation:aws-java:aws-java-sdk-2.2": 220.7, + ":dd-java-agent:instrumentation:mule-4.5": 217.1, + ":dd-java-agent:instrumentation:grpc-1.5": 196.7, + ":dd-java-agent:instrumentation:junit:junit-5:junit-5.3": 195.4, + ":dd-java-agent:instrumentation:liberty:liberty-20.0": 187.3, + ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.8": 175.4, + ":dd-java-agent:instrumentation:maven:maven-3.2.1": 172.2, + ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0": 171.3, + ":dd-java-agent:instrumentation:couchbase:couchbase-3.2": 165.8, + ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7": 159.7, + ":dd-java-agent:instrumentation:aws-java:aws-java-sqs-1.0": 152.1, + ":dd-java-agent:instrumentation:couchbase:couchbase-3.1": 151.2, + ":dd-java-agent:instrumentation:aws-java:aws-java-sqs-2.0": 148.7, + ":dd-java-agent:instrumentation:vertx:vertx-web:vertx-web-4.0": 140.9, + ":dd-java-agent:instrumentation:spark:spark_2.12": 113.9, + ":dd-java-agent:instrumentation:aws-java:aws-java-sdk-1.11": 104.0, + ":dd-java-agent:instrumentation:hibernate:hibernate-core-3.3": 93.6, + ":dd-java-agent:instrumentation:jersey:jersey-2.0": 84.8, + ":dd-java-agent:instrumentation:couchbase:couchbase-2.0": 82.5, + ":dd-java-agent:instrumentation:akka:akka-actor-2.5": 66.8, + ":dd-java-agent:instrumentation:jetty:jetty-server:jetty-server-9.4.21": 64.6, + ":dd-java-agent:instrumentation:lettuce:lettuce-4.0": 47.3, + ":dd-java-agent:instrumentation:jedis:jedis-3.0": 42.0, + ":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-3:mongo-driver-3.1": 34.7, + ":dd-java-agent:instrumentation:elasticsearch:elasticsearch-rest:elasticsearch-rest-7.0": 31.7, + ":dd-java-agent:instrumentation:couchbase:couchbase-2.6": 25.6, + ":dd-java-agent:instrumentation:synapse-3.0": 24.5, + ":dd-java-agent:instrumentation:spring:spring-webmvc:spring-webmvc-5.3": 21.9, + ":dd-java-agent:instrumentation:spring:spring-boot-1.3": 16.6, + ":dd-java-agent:instrumentation:vertx:vertx-web:vertx-web-3.9": 12.6 + } +} diff --git a/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt b/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt new file mode 100644 index 00000000000..9765a436492 --- /dev/null +++ b/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt @@ -0,0 +1,169 @@ +package datadog.gradle.plugin.ci + +import datadog.gradle.plugin.ci.TestShardBalancer.DEFAULT_WEIGHT_SECONDS +import datadog.gradle.plugin.ci.TestShardBalancer.assignSlots +import datadog.gradle.plugin.ci.TestShardBalancer.loadDefaultWeights +import datadog.gradle.plugin.ci.TestShardBalancer.loadWeights +import datadog.gradle.plugin.ci.TestShardBalancer.weightOf +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import java.io.File +import java.util.TreeSet +import kotlin.random.Random + +class TestShardBalancerTest { + + @ParameterizedTest(name = "{0} slots") + @ValueSource(ints = [1, 2, 4, 6, 8, 12]) + fun `every project is selected by exactly one slot`(totalSlots: Int) { + val assignment = assignSlots(PROJECT_PATHS, totalSlots, WEIGHTS) + + val selectedBySlot = (1..totalSlots).map { slot -> PROJECT_PATHS.filter { assignment[it] == slot } } + assertThat(selectedBySlot.flatten()).containsExactlyInAnyOrderElementsOf(PROJECT_PATHS) + } + + @ParameterizedTest(name = "{0} slots") + @ValueSource(ints = [1, 2, 4, 6, 8, 12]) + fun `assigned slots are within range`(totalSlots: Int) { + val assignment = assignSlots(PROJECT_PATHS, totalSlots, WEIGHTS) + + assertThat(assignment.values).allSatisfy { slot -> assertThat(slot).isBetween(1, totalSlots) } + } + + @Test + fun `assignment does not depend on input order`() { + val expected = assignSlots(PROJECT_PATHS, 8, WEIGHTS) + + assertThat(assignSlots(PROJECT_PATHS, 8, WEIGHTS)).isEqualTo(expected) + for (seed in 1..5) { + assertThat(assignSlots(PROJECT_PATHS.shuffled(Random(seed)), 8, WEIGHTS)).isEqualTo(expected) + } + } + + @Test + fun `instrumentation shards are balanced by duration`() { + val instrumentationPaths = PROJECT_PATHS.filter { it.startsWith(INSTRUMENTATION_PREFIX) } + assertThat(instrumentationPaths).hasSizeGreaterThan(300) + + val assignment = assignSlots(PROJECT_PATHS, 8, WEIGHTS) + + val loads = DoubleArray(8) + instrumentationPaths.forEach { loads[assignment.getValue(it) - 1] += weightOf(it, WEIGHTS) } + assertThat(loads.min()).isPositive() + assertThat(loads.max() / loads.min()).isLessThanOrEqualTo(1.25) + } + + @ParameterizedTest(name = "{0} slots") + @ValueSource(ints = [2, 4, 6, 8, 12]) + fun `two heaviest projects never share a slot`(totalSlots: Int) { + // Each one sets a floor on its shard's duration, so sharing a slot would stack the floors + val (heaviest, secondHeaviest) = heaviestProjects(2) + + val assignment = assignSlots(PROJECT_PATHS, totalSlots, WEIGHTS) + + assertThat(assignment.getValue(heaviest)).isNotEqualTo(assignment.getValue(secondHeaviest)) + } + + @Test + fun `heaviest projects land in different slots`() { + val assignment = assignSlots(PROJECT_PATHS, 8, WEIGHTS) + + assertThat(heaviestProjects(8).map { assignment.getValue(it) }).doesNotHaveDuplicates() + } + + @Test + fun `unknown projects receive the default weight`() { + assertThat(weightOf(":unknown", mapOf(":known" to 1.5))).isEqualTo(DEFAULT_WEIGHT_SECONDS) + assertThat(weightOf(":known", mapOf(":known" to 1.5))).isEqualTo(1.5) + + // A known project worth two default weights is balanced by two unknown projects + val assignment = assignSlots( + listOf(":unknown-a", ":known", ":unknown-b"), + 2, + mapOf(":known" to 2 * DEFAULT_WEIGHT_SECONDS) + ) + assertThat(assignment).isEqualTo(mapOf(":known" to 1, ":unknown-a" to 2, ":unknown-b" to 2)) + } + + @Test + fun `single slot selects every project`() { + val assignment = assignSlots(PROJECT_PATHS, 1, WEIGHTS) + + assertThat(assignment).hasSize(PROJECT_PATHS.size) + assertThat(assignment.values).containsOnly(1) + } + + @Test + fun `empty project list yields an empty assignment`() { + assertThat(assignSlots(emptyList(), 8, WEIGHTS)).isEmpty() + } + + @Test + fun `non positive slot count is rejected`() { + assertThatThrownBy { assignSlots(PROJECT_PATHS, 0, WEIGHTS) } + .isInstanceOf(IllegalArgumentException::class.java) + } + + @Test + fun `bundled weights file is parsed`() { + assertThat(WEIGHTS).hasSize(36) + assertThat(WEIGHTS.values).allSatisfy { weight -> assertThat(weight).isPositive() } + // docs/ci-test-shard-weights.md names the two heaviest projects + assertThat(heaviestProjects(2)).containsExactly(JDBC, LETTUCE_5_0) + } + + @Test + fun `every weighted path is an included project`() { + assertThat(INCLUDED_PATHS).containsAll(WEIGHTS.keys) + } + + @Test + fun `only the weights object is read`() { + val weights = loadWeights("""{"_comment": ["c"], "_metadata": {"projects_measured": 1}, "weights": {":a": 1.5}}""") + + assertThat(weights).isEqualTo(mapOf(":a" to 1.5)) + } + + @Test + fun `invalid weights are rejected`() { + assertThatThrownBy { loadWeights("""{"_comment": []}""") } + .isInstanceOf(IllegalArgumentException::class.java) + assertThatThrownBy { loadWeights("""{"weights": {":a": -1}}""") } + .isInstanceOf(IllegalArgumentException::class.java) + assertThatThrownBy { loadWeights("""{"weights": {":a": "slow"}}""") } + .isInstanceOf(IllegalArgumentException::class.java) + } + + companion object { + private const val INSTRUMENTATION_PREFIX = ":dd-java-agent:instrumentation:" + private const val JDBC = ":dd-java-agent:instrumentation:jdbc" + private const val LETTUCE_5_0 = ":dd-java-agent:instrumentation:lettuce:lettuce-5.0" + + private val WEIGHTS = loadDefaultWeights() + + /** Project paths included by the repository settings. */ + private val INCLUDED_PATHS: List = run { + val settings = File(System.getProperty("user.dir")).parentFile.resolve("settings.gradle.kts") + val includePattern = Regex("\"(:[^\"]+)\"") + settings.readLines() + .filterNot { it.trimStart().startsWith("//") } + .flatMap { line -> includePattern.findAll(line).map { it.groupValues[1] }.toList() } + } + + /** Included project paths, plus the implicit parent projects Gradle creates. */ + private val PROJECT_PATHS: List = run { + val paths = TreeSet(listOf(":")) + INCLUDED_PATHS.forEach { include -> + val segments = include.removePrefix(":").split(":") + segments.indices.forEach { i -> paths.add(":" + segments.subList(0, i + 1).joinToString(":")) } + } + paths.toList() + } + + private fun heaviestProjects(count: Int): List = + WEIGHTS.entries.sortedByDescending { it.value }.take(count).map { it.key } + } +} diff --git a/buildSrc/src/test/kotlin/datadog/gradle/plugin/muzzle/MuzzleActionTest.kt b/buildSrc/src/test/kotlin/datadog/gradle/plugin/muzzle/MuzzleActionTest.kt new file mode 100644 index 00000000000..8e05501d619 --- /dev/null +++ b/buildSrc/src/test/kotlin/datadog/gradle/plugin/muzzle/MuzzleActionTest.kt @@ -0,0 +1,57 @@ +package datadog.gradle.plugin.muzzle + +import datadog.gradle.plugin.muzzle.MuzzleAction.Companion.bootAndToolClassLoaders +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File +import java.util.concurrent.atomic.AtomicLong + +class MuzzleActionTest { + + @TempDir + lateinit var tempDir: File + + @Test + fun `bootstrap classloader only exposes the requested class-path`() { + val base = classPathEntry("base") + val otel = classPathEntry("otel") + val tooling = listOf(classPathEntry("tooling")) + val buildStamp = nextBuildStamp() + + // A larger, unrelated bootstrap class-path checked first in the same build, like + // graal-native-image-20.0 adding agent-installer before opentelemetry-1.47 adds otel-bootstrap + bootAndToolClassLoaders(buildStamp, listOf(base, classPathEntry("installer"), classPathEntry("installer-deps")), tooling) + val (bootCL, toolCL) = bootAndToolClassLoaders(buildStamp, listOf(base, otel), tooling) + + assertThat(bootCL.getResource("otel.marker")).isNotNull() + assertThat(bootCL.getResource("installer.marker")).isNull() + assertThat(toolCL.getResource("otel.marker")).isNotNull() + assertThat(toolCL.getResource("tooling.marker")).isNotNull() + } + + @Test + fun `classloaders are reused for the same class-path within a build`() { + val bootstrap = listOf(classPathEntry("base")) + val tooling = listOf(classPathEntry("tooling")) + val buildStamp = nextBuildStamp() + + val classLoaders = bootAndToolClassLoaders(buildStamp, bootstrap, tooling) + + assertThat(bootAndToolClassLoaders(buildStamp, bootstrap, tooling)).isSameAs(classLoaders) + assertThat(bootAndToolClassLoaders(nextBuildStamp(), bootstrap, tooling)).isNotSameAs(classLoaders) + } + + /** Creates a class-path directory holding a `.marker` resource. */ + private fun classPathEntry(name: String): File = + File(tempDir, name).apply { + mkdirs() + File(this, "$name.marker").writeText(name) + } + + companion object { + private val buildStamps = AtomicLong(System.currentTimeMillis()) + + private fun nextBuildStamp() = buildStamps.incrementAndGet() + } +} diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-annotations-1.20/build.gradle b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-annotations-1.20/build.gradle index 3811b255e1e..55b3bf4fddb 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-annotations-1.20/build.gradle +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-annotations-1.20/build.gradle @@ -21,6 +21,10 @@ dependencies { compileOnly group: 'com.google.auto.value', name: 'auto-value-annotations', version: '1.6.6' implementation project(':dd-java-agent:agent-otel:otel-shim') + + // otel-shim is compiled against otel-bootstrap classes (e.g. OtelInstrument) that the agent loads on the bootstrap + // class-path, so muzzle checks this module against the same bootstrap classes, like the sibling opentelemetry modules. + muzzleBootstrap project(path: ':dd-java-agent:agent-otel:otel-bootstrap', configuration: 'shadow') testImplementation group: 'io.opentelemetry', name: 'opentelemetry-api', version: openTelemetryVersion testImplementation group: 'io.opentelemetry.instrumentation', name: 'opentelemetry-instrumentation-annotations', version: openTelemetryVersion latest1xDepTestImplementation group: 'io.opentelemetry.instrumentation', name: 'opentelemetry-instrumentation-annotations', version: '1+' diff --git a/docs/ci-test-shard-weights.md b/docs/ci-test-shard-weights.md new file mode 100644 index 00000000000..b805040c554 --- /dev/null +++ b/docs/ci-test-shard-weights.md @@ -0,0 +1,94 @@ +# CI test shard weights + +CI test jobs are split across parallel shards with `-Pslot=X/Y` (for example `test_inst` runs 8 shards). +Each Gradle project is assigned to exactly one shard by +[`TestShardBalancer`](../buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/TestShardBalancer.kt), which +packs projects by test duration rather than by project count, so that shards finish at about the same time. + +## The weights file + +[`test-shard-weights.json`](../buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json) +maps Gradle project paths to the seconds of test execution each project costs per pipeline run in the `test_inst` +job. Only the `weights` object is read; `_comment` and `_metadata` document where the numbers come from. + +Projects absent from the file get `TestShardBalancer.DEFAULT_WEIGHT_SECONDS` (20.6 seconds, the mean weight of the +measured projects that are left out of the file). The file therefore only needs to list the projects whose tests are +much heavier than average, but it must not miss any of the heaviest ones: a heavy project that falls back to the +default weight can overload whichever shard it lands in. + +Projects are then placed heaviest first (ties broken by project path) into the least loaded shard (ties broken by +lowest shard index). The assignment is computed from all project paths, so it does not depend on which shard is +running, and it does not use hash codes, so it is the same on every JVM and machine. + +## Regenerating the weights + +Weights come from Datadog CI Visibility test events at the **module** level, over a 7-day window: + +``` +@git.repository.id_v2:"github.com/datadog/dd-trace-java" @test.configuration.job_name:test_inst @test.configuration.jvm:11 +``` + +1. Aggregate `sum(@duration)` grouped by `@test.module`. Module names come from the uploaded JUnit reports and look + like `__TEST-.xml`, e.g. `akka-actor-2.5_test_TEST-AkkaActorTest.xml`. +2. Strip the `__TEST-.xml` suffix to get the project name, and sum all modules of a project + (all of its test tasks and test classes). Map the project name to its Gradle project path from + `settings.gradle.kts`, e.g. `jdbc` to `:dd-java-agent:instrumentation:jdbc`. Watch out for project names that + contain underscores, such as `spark_2.12`. +3. Count the observed pipelines with `cardinality(@ci.pipeline.id)` over the same query. +4. Divide each project's total by the pipeline count, and convert from nanoseconds to seconds. The result is a mean + per pipeline, not a median. + +For example, `jdbc` summed to 548,323.5 seconds over 491 pipelines, giving a weight of `1116.7`. + +Measure every project listed in `settings.gradle.kts` one by one (e.g. `@test.module:lettuce-5.0_*`) instead of +reading a single grouped aggregation or guessing family prefixes: + +- grouped aggregations can silently return only the first few buckets; +- a prefix that matches nothing (e.g. `aws-sdk*` when the projects are named `aws-java-sdk-*`) means the prefix is + wrong, not that the project is light. + +Keep the projects that clearly dominate the total, and set `DEFAULT_WEIGHT_SECONDS` to the mean weight of the +projects left out. Update `_metadata` (`generated`, `pipelines_observed`, `projects_measured`) and the window end +date in `_comment`, update the expected entry count in `TestShardBalancerTest`, then run the buildSrc tests: + +```shell +./gradlew :buildSrc:test -PrunBuildSrcTests +``` + +`TestShardBalancerTest` checks that every weighted path is a project from `settings.gradle.kts`, that the two +heaviest projects land in different shards, and that the modeled instrumentation shards stay balanced, with a +max/min load ratio of at most 1.25. That ratio is computed from the weights, not measured, see below. + +## Known limitation: the heaviest project sets a floor + +A project's tests always run in a single shard, and tests run in parallel within a shard. The balancer only adds up +seconds, so it cannot see that a shard dominated by one large project finishes later than a shard with the same total +spread over many small projects. In the first pipeline using this balancer, shards 1/8 and 4/8 ran almost the same +total test time (2312 s and 2292 s), yet took 39 and 24 minutes: shard 1/8 held `jdbc`. + +The two heaviest projects per pipeline are currently: + +- `:dd-java-agent:instrumentation:jdbc`: 1116.7 s, about 18.6 minutes; +- `:dd-java-agent:instrumentation:lettuce:lettuce-5.0`: 978.1 s, about 16.3 minutes. + +The slowest shard can therefore never be faster than the heaviest single project, plus job setup. A project's +wall-clock span is also longer than its summed test time: in that first pipeline, the `jdbc` tests (20.5 minutes of +summed test time) spanned 27.6 minutes, starting about 10 minutes into the job, so its shard took 39 minutes. Under +the previous hash-based assignment, the `jdbc` shard was already the slowest one. + +The modeled max/min ratio is optimistic whenever one project approaches the target shard time, so treat the 1.25 +bound as a check of the model rather than a guarantee for wall-clock time. + +The two heaviest projects must not share a shard, otherwise their floors stack. Capping weights would not help: +under-weighting `jdbc` makes the model look balanced while letting more work pile onto the `jdbc` shard. Lowering the +floor needs finer-grained sharding than whole projects. + +## When to refresh + +- After adding, splitting, or renaming a heavy instrumentation module, or one that adds many slow tests. +- When CI Visibility shows the `test_inst` shard durations drifting apart again. + +Stale weights only make the balance worse. They never drop test coverage: every project is always assigned to exactly +one shard, and projects missing from the file (including new or renamed ones) fall back to the default weight. +After renaming or removing a weighted project, update its entry: `TestShardBalancerTest` fails on weights for paths +that are not projects.