From 382d1065350a14ca989d6bafae7b6faa1d8dec25 Mon Sep 17 00:00:00 2001 From: Charles Cote Date: Mon, 12 Oct 2026 17:33:46 +0200 Subject: [PATCH 1/2] Split Lettuce tests across CI shards --- .../gradle/plugin/ci/test-shard-weights.json | 9 ++++-- .../gradle/plugin/ci/TestShardBalancerTest.kt | 7 +++-- dd-java-agent/instrumentation/build.gradle | 6 ++++ .../lettuce/lettuce-5.0/build.gradle | 28 +++++++++++++++++++ .../lettuce-5.0/lettuce-51-test/build.gradle | 1 + .../lettuce-5.0/lettuce-60-test/build.gradle | 1 + .../lettuce-5.0/lettuce-61-test/build.gradle | 1 + .../lettuce-5.0/lettuce-62-test/build.gradle | 1 + .../lettuce-base-test/build.gradle | 1 + .../lettuce-forked-tests/build.gradle | 1 + docs/ci-test-shard-weights.md | 20 ++++++++++--- settings.gradle.kts | 6 ++++ 12 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-51-test/build.gradle create mode 100644 dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-60-test/build.gradle create mode 100644 dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-61-test/build.gradle create mode 100644 dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-62-test/build.gradle create mode 100644 dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-base-test/build.gradle create mode 100644 dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-forked-tests/build.gradle diff --git a/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json b/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json index 776ded8e17c..293a1f80ead 100644 --- a/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json +++ b/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json @@ -17,10 +17,9 @@ "job": "test_inst", "pipelines_observed": 491, "projects_measured": 35, - "projects_derived_from_measured_suites": 14 + "projects_derived_from_measured_suites": 20 }, "weights": { - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0": 978.1, ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0": 612.3, ":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0": 509.4, ":dd-java-agent:instrumentation:kafka:kafka-clients-0.11": 379.1, @@ -32,6 +31,7 @@ ":dd-java-agent:instrumentation:jdbc:jdbc-remote-dbm-prepared-forked-test": 228.78, ":dd-java-agent:instrumentation:aws-java:aws-java-sdk-2.2": 220.7, ":dd-java-agent:instrumentation:mule-4.5": 217.1, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-forked-tests": 214.1, ":dd-java-agent:instrumentation:grpc-1.5": 196.7, ":dd-java-agent:instrumentation:junit:junit-5:junit-5.3": 195.4, ":dd-java-agent:instrumentation:liberty:liberty-20.0": 187.3, @@ -42,9 +42,14 @@ ":dd-java-agent:instrumentation:jdbc:jdbc-old-h2-test": 167.347, ":dd-java-agent:instrumentation:couchbase:couchbase-3.2": 165.8, ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7": 159.7, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-60-test": 156.7, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-61-test": 156.7, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-62-test": 156.6, ":dd-java-agent:instrumentation:aws-java:aws-java-sqs-1.0": 152.1, ":dd-java-agent:instrumentation:couchbase:couchbase-3.1": 151.2, ":dd-java-agent:instrumentation:aws-java:aws-java-sqs-2.0": 148.7, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-51-test": 147.0, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-base-test": 147.0, ":dd-java-agent:instrumentation:vertx:vertx-web:vertx-web-4.0": 140.9, ":dd-java-agent:instrumentation:jdbc:jdbc-test": 139.615, ":dd-java-agent:instrumentation:spark:spark_2.12": 113.9, diff --git a/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt b/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt index 6966760302d..1e0bf917005 100644 --- a/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt +++ b/buildSrc/src/test/kotlin/datadog/gradle/plugin/ci/TestShardBalancerTest.kt @@ -109,9 +109,9 @@ class TestShardBalancerTest { @Test fun `bundled weights file is parsed`() { - assertThat(WEIGHTS).hasSize(49) + assertThat(WEIGHTS).hasSize(54) assertThat(WEIGHTS.values).allSatisfy { weight -> assertThat(weight).isPositive() } - assertThat(heaviestProjects(2)).containsExactly(LETTUCE_5_0, AKKA_HTTP_10_0) + assertThat(heaviestProjects(2)).containsExactly(AKKA_HTTP_10_0, SPRING_WEBFLUX_5_0) } @Test @@ -138,8 +138,9 @@ class TestShardBalancerTest { companion object { private const val INSTRUMENTATION_PREFIX = ":dd-java-agent:instrumentation:" - private const val LETTUCE_5_0 = ":dd-java-agent:instrumentation:lettuce:lettuce-5.0" private const val AKKA_HTTP_10_0 = ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0" + private const val SPRING_WEBFLUX_5_0 = + ":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0" private val WEIGHTS = loadDefaultWeights() diff --git a/dd-java-agent/instrumentation/build.gradle b/dd-java-agent/instrumentation/build.gradle index 534ba799ed5..7cc6b761339 100644 --- a/dd-java-agent/instrumentation/build.gradle +++ b/dd-java-agent/instrumentation/build.gradle @@ -102,6 +102,12 @@ instrumentationNaming { "jdbc-remote-dbm-prepared-forked-test", "jdbc-remote-instrumentation-forked-test", "jdbc-other-forked-tests", + "lettuce-base-test", + "lettuce-forked-tests", + "lettuce-51-test", + "lettuce-60-test", + "lettuce-61-test", + "lettuce-62-test", ] suffixes = ["-common", "-stubs", "-iast"] } diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/build.gradle b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/build.gradle index 8c75a2232d6..63257313872 100644 --- a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/build.gradle +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/build.gradle @@ -49,3 +49,31 @@ dependencies { usesService(testcontainersLimit) } } + +// In sharded CI, virtual child projects expose each version suite independently. The latest +// dependency suites stay on this project and continue to run through allLatestDepTests. +if (rootProject.providers.gradleProperty("slot").isPresent()) { + tasks.named("allTests") { + setDependsOn([]) + } +} + +def ciTestShards = [ + 'lettuce-base-test': tasks.named('test'), + 'lettuce-forked-tests': tasks.named('forkedTest'), + 'lettuce-51-test': tasks.named('lettuce51Test'), + 'lettuce-60-test': tasks.named('lettuce60Test'), + 'lettuce-61-test': tasks.named('lettuce61Test'), + 'lettuce-62-test': tasks.named('lettuce62Test'), +] + +def ciShardProjectPaths = rootProject.extensions.extraProperties.get('ciShardProjectPaths') as Map +ciTestShards.each { childName, testTask -> + def childPath = ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:$childName" + ciShardProjectPaths[testTask.get().path] = childPath + project(childPath) { + tasks.register("allTests") { + dependsOn(testTask) + } + } +} diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-51-test/build.gradle b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-51-test/build.gradle new file mode 100644 index 00000000000..263291b4eb9 --- /dev/null +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-51-test/build.gradle @@ -0,0 +1 @@ +// CI-only virtual project; its allTests task is registered by the parent Lettuce 5 project. diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-60-test/build.gradle b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-60-test/build.gradle new file mode 100644 index 00000000000..263291b4eb9 --- /dev/null +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-60-test/build.gradle @@ -0,0 +1 @@ +// CI-only virtual project; its allTests task is registered by the parent Lettuce 5 project. diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-61-test/build.gradle b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-61-test/build.gradle new file mode 100644 index 00000000000..263291b4eb9 --- /dev/null +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-61-test/build.gradle @@ -0,0 +1 @@ +// CI-only virtual project; its allTests task is registered by the parent Lettuce 5 project. diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-62-test/build.gradle b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-62-test/build.gradle new file mode 100644 index 00000000000..263291b4eb9 --- /dev/null +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-62-test/build.gradle @@ -0,0 +1 @@ +// CI-only virtual project; its allTests task is registered by the parent Lettuce 5 project. diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-base-test/build.gradle b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-base-test/build.gradle new file mode 100644 index 00000000000..263291b4eb9 --- /dev/null +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-base-test/build.gradle @@ -0,0 +1 @@ +// CI-only virtual project; its allTests task is registered by the parent Lettuce 5 project. diff --git a/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-forked-tests/build.gradle b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-forked-tests/build.gradle new file mode 100644 index 00000000000..263291b4eb9 --- /dev/null +++ b/dd-java-agent/instrumentation/lettuce/lettuce-5.0/lettuce-forked-tests/build.gradle @@ -0,0 +1 @@ +// CI-only virtual project; its allTests task is registered by the parent Lettuce 5 project. diff --git a/docs/ci-test-shard-weights.md b/docs/ci-test-shard-weights.md index 95ba8ff6212..4823fc5b40c 100644 --- a/docs/ci-test-shard-weights.md +++ b/docs/ci-test-shard-weights.md @@ -77,6 +77,18 @@ The largest measured JDBC unit after this split is the old H2 remote suite at 32 original 1116.7-second JDBC project. The next largest units are the remote DBM fork at 310.736 s and the remote instrumentation fork at 279.971 s. The checked-in weights come directly from that pipeline's suite durations. +## Lettuce 5 suite shards + +The Lettuce 5 instrumentation uses six virtual child projects for its main `test_inst` suites: base, forked, 5.1, +6.0, 6.1, and 6.2. Each child delegates to the existing task in `lettuce-5.0`; all suites retain their Redis +container limit. The separate `allLatestDepTests` path remains on the parent project and continues to run +`latestDepTest` and `latestDepForkedTest` unchanged. + +The measured project weight is 978.1 s. Available suite aggregates give 147 s for the base task and 470 s for the +three 6.x tasks. The initial weights assign 156.7 s, 156.7 s, and 156.6 s to 6.0, 6.1, and 6.2; 147 s to 5.1; and +the remaining 214.1 s to the forked task. Replace these provisional weights with direct suite measurements after +the first coverage-complete CI run. + ## Known limitation: the heaviest project sets a floor A project's tests always run in a single shard, and tests run in parallel within a shard. The balancer only adds up @@ -89,9 +101,9 @@ Before the JDBC suite split, the two heaviest projects per pipeline were: - `:dd-java-agent:instrumentation:jdbc`: 1116.7 s, about 18.6 minutes; - `:dd-java-agent:instrumentation:lettuce:lettuce-5.0`: 978.1 s, about 16.3 minutes. -After the split, JDBC no longer sets that floor; `lettuce-5.0` is the heaviest remaining project. The slowest shard -can never be faster than the heaviest single project, plus job setup. A project's wall-clock span is also longer -than its summed test time: in that first pipeline, the `jdbc` tests (20.5 minutes of summed test time) spanned 27.6 +After the JDBC and Lettuce splits, their largest independently scheduled units are 316.5 s and 214.1 s. The largest +remaining project is `akka-http-10.0` at 612.3 s. A project's wall-clock span is also longer than its summed test +time: in the first duration-aware pipeline, the unsplit `jdbc` tests (20.5 minutes of summed test time) spanned 27.6 minutes, starting about 10 minutes into the job, so its shard took 39 minutes. Under the previous hash-based assignment, the `jdbc` shard was already the slowest one. @@ -100,7 +112,7 @@ bound as a check of the model rather than a guarantee for wall-clock time. The two heaviest projects must not share a shard, otherwise their floors stack. Capping weights only makes the model look balanced while letting more work pile onto the affected shard. Lowering the floor requires splitting the work -into independently scheduled units, as the JDBC suite shards do above. +into independently scheduled units, as the JDBC and Lettuce suite shards do above. ## When to refresh diff --git a/settings.gradle.kts b/settings.gradle.kts index 1cf2e47fb42..58c1fa27f0a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -491,6 +491,12 @@ include( ":dd-java-agent:instrumentation:karate:karate-2.0", ":dd-java-agent:instrumentation:kotlin-coroutines-1.3", ":dd-java-agent:instrumentation:lettuce:lettuce-4.0", + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-base-test", + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-forked-tests", + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-51-test", + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-60-test", + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-61-test", + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-62-test", ":dd-java-agent:instrumentation:lettuce:lettuce-5.0", ":dd-java-agent:instrumentation:liberty:liberty-20.0", ":dd-java-agent:instrumentation:liberty:liberty-23.0", From 3f933c0e82ecb274ba9dc80f2d931df95732dae1 Mon Sep 17 00:00:00 2001 From: Charles Cote Date: Mon, 12 Oct 2026 20:15:07 +0200 Subject: [PATCH 2/2] Tune Lettuce shard weights from CI --- .../gradle/plugin/ci/test-shard-weights.json | 13 +++++++------ docs/ci-test-shard-weights.md | 10 +++++----- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json b/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json index 293a1f80ead..e96e829b060 100644 --- a/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json +++ b/buildSrc/src/main/resources/datadog/gradle/plugin/ci/test-shard-weights.json @@ -7,6 +7,7 @@ "divided by the 491 observed pipelines.", "JDBC virtual-project weights use coverage-complete test results from", "pipeline 144158285 on 2026-10-12.", + "Lettuce virtual-project weights use coverage-complete pipeline 144154452.", "Projects absent from this map fall back to the default weight.", "See docs/ci-test-shard-weights.md to regenerate." ], @@ -24,17 +25,22 @@ ":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0": 509.4, ":dd-java-agent:instrumentation:kafka:kafka-clients-0.11": 379.1, ":dd-java-agent:instrumentation:jdbc:jdbc-remote-old-h2-test": 321.831, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-62-test": 318.271, ":dd-java-agent:instrumentation:jdbc:jdbc-remote-dbm-forked-test": 310.736, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-51-test": 290.357, ":dd-java-agent:instrumentation:jdbc:jdbc-remote-instrumentation-forked-test": 279.971, ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.6": 258.0, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-60-test": 253.057, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-base-test": 244.848, ":dd-java-agent:instrumentation:jdbc:jdbc-remote-old-postgres-test": 241.128, ":dd-java-agent:instrumentation:jdbc:jdbc-remote-dbm-prepared-forked-test": 228.78, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-61-test": 222.911, ":dd-java-agent:instrumentation:aws-java:aws-java-sdk-2.2": 220.7, ":dd-java-agent:instrumentation:mule-4.5": 217.1, - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-forked-tests": 214.1, ":dd-java-agent:instrumentation:grpc-1.5": 196.7, ":dd-java-agent:instrumentation:junit:junit-5:junit-5.3": 195.4, ":dd-java-agent:instrumentation:liberty:liberty-20.0": 187.3, + ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-forked-tests": 180.254, ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.8": 175.4, ":dd-java-agent:instrumentation:jdbc:jdbc-old-postgres-test": 173.218, ":dd-java-agent:instrumentation:maven:maven-3.2.1": 172.2, @@ -42,14 +48,9 @@ ":dd-java-agent:instrumentation:jdbc:jdbc-old-h2-test": 167.347, ":dd-java-agent:instrumentation:couchbase:couchbase-3.2": 165.8, ":dd-java-agent:instrumentation:rabbitmq-amqp-2.7": 159.7, - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-60-test": 156.7, - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-61-test": 156.7, - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-62-test": 156.6, ":dd-java-agent:instrumentation:aws-java:aws-java-sqs-1.0": 152.1, ":dd-java-agent:instrumentation:couchbase:couchbase-3.1": 151.2, ":dd-java-agent:instrumentation:aws-java:aws-java-sqs-2.0": 148.7, - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-51-test": 147.0, - ":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-base-test": 147.0, ":dd-java-agent:instrumentation:vertx:vertx-web:vertx-web-4.0": 140.9, ":dd-java-agent:instrumentation:jdbc:jdbc-test": 139.615, ":dd-java-agent:instrumentation:spark:spark_2.12": 113.9, diff --git a/docs/ci-test-shard-weights.md b/docs/ci-test-shard-weights.md index 4823fc5b40c..c2380aeeddc 100644 --- a/docs/ci-test-shard-weights.md +++ b/docs/ci-test-shard-weights.md @@ -84,10 +84,10 @@ The Lettuce 5 instrumentation uses six virtual child projects for its main `test container limit. The separate `allLatestDepTests` path remains on the parent project and continues to run `latestDepTest` and `latestDepForkedTest` unchanged. -The measured project weight is 978.1 s. Available suite aggregates give 147 s for the base task and 470 s for the -three 6.x tasks. The initial weights assign 156.7 s, 156.7 s, and 156.6 s to 6.0, 6.1, and 6.2; 147 s to 5.1; and -the remaining 214.1 s to the forked task. Replace these provisional weights with direct suite measurements after -the first coverage-complete CI run. +Pipeline `144154452` provided a coverage-complete JVM 11 measurement: 35 suites and 314 passing tests. Its measured +task weights are 318.271 s for 6.2, 290.357 s for 5.1, 253.057 s for 6.0, 244.848 s for the base task, 222.911 s +for 6.1, and 180.254 s for the forked task. The largest independently scheduled Lettuce unit is therefore 67.5% +smaller than the original 978.1-second project. ## Known limitation: the heaviest project sets a floor @@ -101,7 +101,7 @@ Before the JDBC suite split, the two heaviest projects per pipeline were: - `:dd-java-agent:instrumentation:jdbc`: 1116.7 s, about 18.6 minutes; - `:dd-java-agent:instrumentation:lettuce:lettuce-5.0`: 978.1 s, about 16.3 minutes. -After the JDBC and Lettuce splits, their largest independently scheduled units are 316.5 s and 214.1 s. The largest +After the JDBC and Lettuce splits, their largest independently scheduled units are 321.831 s and 318.271 s. The largest remaining project is `akka-http-10.0` at 612.3 s. A project's wall-clock span is also longer than its summed test time: in the first duration-aware pipeline, the unsplit `jdbc` tests (20.5 minutes of summed test time) spanned 27.6 minutes, starting about 10 minutes into the job, so its shard took 39 minutes. Under the previous hash-based