Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
"JDBC virtual-project weights use coverage-complete test results from",
"pipeline 144158285 on 2026-10-12.",
"Lettuce virtual-project weights use coverage-complete pipeline 144154452.",
"Akka HTTP and WebFlux virtual-project weights use pipeline 144209984.",
"Projects absent from this map fall back to the default weight.",
"See docs/ci-test-shard-weights.md to regenerate."
],
Expand All @@ -18,11 +19,9 @@
"job": "test_inst",
"pipelines_observed": 491,
"projects_measured": 35,
"projects_derived_from_measured_suites": 20
"projects_derived_from_measured_suites": 31
},
"weights": {
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0": 612.3,
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0": 509.4,
":dd-java-agent:instrumentation:kafka:kafka-clients-0.11": 379.1,
":dd-java-agent:instrumentation:jdbc:jdbc-remote-old-h2-test": 321.831,
":dd-java-agent:instrumentation:lettuce:lettuce-5.0:lettuce-62-test": 318.271,
Expand All @@ -46,15 +45,25 @@
":dd-java-agent:instrumentation:maven:maven-3.2.1": 172.2,
":dd-java-agent:instrumentation:mongo:mongo-driver:mongo-driver-4.0": 171.3,
":dd-java-agent:instrumentation:jdbc:jdbc-old-h2-test": 167.347,
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0:akka-http-version-102-tests": 166.919,
":dd-java-agent:instrumentation:couchbase:couchbase-3.2": 165.8,
":dd-java-agent:instrumentation:rabbitmq-amqp-2.7": 159.7,
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0:akka-http-base-forked-tests": 158.884,
":dd-java-agent:instrumentation:aws-java:aws-java-sqs-1.0": 152.1,
":dd-java-agent:instrumentation:couchbase:couchbase-3.1": 151.2,
":dd-java-agent:instrumentation:aws-java:aws-java-sqs-2.0": 148.7,
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0:akka-http-version-101-forked-tests": 148.509,
":dd-java-agent:instrumentation:vertx:vertx-web:vertx-web-4.0": 140.9,
":dd-java-agent:instrumentation:jdbc:jdbc-test": 139.615,
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:webflux-client-d-test": 122.91,
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0:akka-http-version-101-tests": 118.529,
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:webflux-forked-iast-tests": 116.325,
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0:akka-http-base-tests": 115.244,
":dd-java-agent:instrumentation:spark:spark_2.12": 113.9,
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:webflux-client-c-test": 111.856,
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:webflux-client-a-test": 104.088,
":dd-java-agent:instrumentation:aws-java:aws-java-sdk-1.11": 104.0,
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:webflux-client-b-test": 102.246,
":dd-java-agent:instrumentation:hibernate:hibernate-core-3.3": 93.6,
":dd-java-agent:instrumentation:jersey:jersey-2.0": 84.8,
":dd-java-agent:instrumentation:couchbase:couchbase-2.0": 82.5,
Expand All @@ -72,6 +81,7 @@
":dd-java-agent:instrumentation:jdbc:jdbc-oracle-forked-tests": 25.523,
":dd-java-agent:instrumentation:synapse-3.0": 24.5,
":dd-java-agent:instrumentation:spring:spring-webmvc:spring-webmvc-5.3": 21.9,
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:webflux-client-other-test": 20.6,
":dd-java-agent:instrumentation:spring:spring-boot-1.3": 16.6,
":dd-java-agent:instrumentation:vertx:vertx-web:vertx-web-3.9": 12.6,
":dd-java-agent:instrumentation:jdbc:jdbc-sql-server-forked-tests": 11.867
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,9 @@ class TestShardBalancerTest {

@Test
fun `bundled weights file is parsed`() {
assertThat(WEIGHTS).hasSize(54)
assertThat(WEIGHTS).hasSize(63)
assertThat(WEIGHTS.values).allSatisfy { weight -> assertThat(weight).isPositive() }
assertThat(heaviestProjects(2)).containsExactly(AKKA_HTTP_10_0, SPRING_WEBFLUX_5_0)
assertThat(heaviestProjects(2)).containsExactly(KAFKA_CLIENTS_0_11, JDBC_REMOTE_OLD_H2_TEST)
}

@Test
Expand All @@ -138,9 +138,10 @@ class TestShardBalancerTest {

companion object {
private const val INSTRUMENTATION_PREFIX = ":dd-java-agent:instrumentation:"
private const val AKKA_HTTP_10_0 = ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0"
private const val SPRING_WEBFLUX_5_0 =
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0"
private const val KAFKA_CLIENTS_0_11 =
":dd-java-agent:instrumentation:kafka:kafka-clients-0.11"
private const val JDBC_REMOTE_OLD_H2_TEST =
":dd-java-agent:instrumentation:jdbc:jdbc-remote-old-h2-test"

private val WEIGHTS = loadDefaultWeights()

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// CI-only virtual project; its allTests task is registered by the parent project.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// CI-only virtual project; its allTests task is registered by the parent project.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// CI-only virtual project; its allTests task is registered by the parent project.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// CI-only virtual project; its allTests task is registered by the parent project.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// CI-only virtual project; its allTests task is registered by the parent project.
Original file line number Diff line number Diff line change
Expand Up @@ -183,3 +183,32 @@ tasks.named("compileVersion102Scala213TestGroovy", GroovyCompile) {
tasks.named("compileLatestDepTestGroovy", GroovyCompile) {
classpath += files(tasks.named('compileLatestDepTestScala').map { it.destinationDirectory })
}

// Split the large Akka HTTP project into independently scheduled CI units. Existing test tasks
// remain unchanged for local builds; only sharded CI delegates them through virtual projects.
if (rootProject.providers.gradleProperty("slot").isPresent()) {
tasks.named("allTests") {
setDependsOn([])
}
}

def ciAkkaTestShards = [
'akka-http-base-tests': [tasks.named('baseTest'), tasks.named('lagomTest')],
'akka-http-base-forked-tests': [tasks.named('baseTestForkedTest'), tasks.named('iastTest')],
'akka-http-version-101-tests': [tasks.named('version101Test'), tasks.named('version101IastTest')],
'akka-http-version-101-forked-tests': [tasks.named('version101ForkedTest'), tasks.named('version102IastTest')],
'akka-http-version-102-tests': [tasks.named('version102Scala213Test')],
]

def ciShardProjectPaths = rootProject.extensions.extraProperties.get('ciShardProjectPaths') as Map
ciAkkaTestShards.each { childName, testTasks ->
def childPath = ":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0:$childName"
testTasks.each { testTask ->
ciShardProjectPaths[testTask.get().path] = childPath
}
project(childPath) {
tasks.register("allTests") {
dependsOn(testTasks)
}
}
}
11 changes: 11 additions & 0 deletions dd-java-agent/instrumentation/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,17 @@ instrumentationNaming {
"lettuce-60-test",
"lettuce-61-test",
"lettuce-62-test",
"akka-http-base-tests",
"akka-http-base-forked-tests",
"akka-http-version-101-tests",
"akka-http-version-101-forked-tests",
"akka-http-version-102-tests",
"webflux-client-a-test",
"webflux-client-b-test",
"webflux-client-c-test",
"webflux-client-d-test",
"webflux-client-other-test",
"webflux-forked-iast-tests",
]
suffixes = ["-common", "-stubs", "-iast"]
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,3 +137,76 @@ tasks.named("latestDepTest", Test) {
dependsOn "latestBoot24Test"
dependsOn "latestBoot2LatestTest"
}

def ciWebfluxBaseGroups = [
webfluxClientATest: [
'**/SpringWebfluxHttpClientBasicTest.class',
'**/SpringWebfluxHttpClientBuilderReuseTest.class',
],
webfluxClientBTest: [
'**/SpringWebfluxHttpClientDoAfterTerminateTest.class',
'**/SpringWebfluxHttpClientDoOnSuccessTest.class',
],
webfluxClientCTest: [
'**/SpringWebfluxHttpClientDoOnTerminateTest.class',
'**/SpringWebfluxHttpClientFilterBuilderReuseTest.class',
],
webfluxClientDTest: [
'**/SpringWebfluxHttpClientFilterTest.class',
'**/SpringWebfluxHttpClientFiltersRemovedTest.class',
],
]

ciWebfluxBaseGroups.each { taskName, includes ->
tasks.register(taskName, Test) {
group = LifecycleBasePlugin.VERIFICATION_GROUP
useJUnitPlatform()
testClassesDirs = sourceSets.test.output.classesDirs
classpath = sourceSets.test.runtimeClasspath
setIncludes(includes)
onlyIf("CI sharding is enabled") { rootProject.providers.gradleProperty("slot").isPresent() }
}
}

tasks.register('webfluxClientOtherTest', Test) {
group = LifecycleBasePlugin.VERIFICATION_GROUP
useJUnitPlatform()
testClassesDirs = sourceSets.test.output.classesDirs
classpath = sourceSets.test.runtimeClasspath
ciWebfluxBaseGroups.values().flatten().each { pattern -> exclude pattern }
failOnNoDiscoveredTests = false
onlyIf("CI sharding is enabled") { rootProject.providers.gradleProperty("slot").isPresent() }
}

// In sharded CI, replace the original base task with disjoint class groups and expose every group
// through a virtual child project. Local allTests and check retain their normal behavior.
if (rootProject.providers.gradleProperty("slot").isPresent()) {
tasks.named("allTests") {
setDependsOn([])
}
tasks.named("test", Test) {
enabled = false
}
}

def ciWebfluxTestShards = [
'webflux-client-a-test': [tasks.named('webfluxClientATest')],
'webflux-client-b-test': [tasks.named('webfluxClientBTest')],
'webflux-client-c-test': [tasks.named('webfluxClientCTest')],
'webflux-client-d-test': [tasks.named('webfluxClientDTest')],
'webflux-client-other-test': [tasks.named('webfluxClientOtherTest')],
'webflux-forked-iast-tests': [tasks.named('forkedTest'), tasks.named('iastTest')],
]

def ciShardProjectPaths = rootProject.extensions.extraProperties.get('ciShardProjectPaths') as Map
ciWebfluxTestShards.each { childName, testTasks ->
def childPath = ":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:$childName"
testTasks.each { testTask ->
ciShardProjectPaths[testTask.get().path] = childPath
}
project(childPath) {
tasks.register("allTests") {
dependsOn(testTasks)
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// CI-only virtual project; its allTests task is registered by the parent project.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// CI-only virtual project; its allTests task is registered by the parent project.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// CI-only virtual project; its allTests task is registered by the parent project.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// CI-only virtual project; its allTests task is registered by the parent project.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// CI-only virtual project; its allTests task is registered by the parent project.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// CI-only virtual project; its allTests task is registered by the parent project.
28 changes: 22 additions & 6 deletions docs/ci-test-shard-weights.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,21 @@ task weights are 318.271 s for 6.2, 290.357 s for 5.1, 253.057 s for 6.0, 244.84
for 6.1, and 180.254 s for the forked task. The largest independently scheduled Lettuce unit is therefore 67.5%
smaller than the original 978.1-second project.

## Akka HTTP and Spring WebFlux suite shards

Akka HTTP 10.0 exposes five virtual projects built from its existing version, forked, IAST, and
Lagom tasks. Pipeline `144209984` measured all 40 JVM 11 suites and 2,044 test executions, while
the Java 8 run supplied the Lagom-only duration. The largest unit is the Scala 2.13 suite at
166.919 s, 72.7% smaller than the original 612.3-second project.

Spring WebFlux 5.0 has one dominant base task containing eight similarly sized client suites. CI
divides those suites into four disjoint two-class tasks, keeps an empty catch-all for newly added
classes, and groups the existing forked and IAST tasks. Pipeline `144209984` measured all 12 JVM 11
suites and 359 test executions. The four base groups took between 102.246 s and 122.910 s, and the
forked/IAST group took 116.325 s. The largest unit is 75.9% smaller than the original 509.4-second
project. Outside sharded CI, the original test tasks remain the local `allTests` and `check` entry
points.

## Known limitation: the heaviest project sets a floor

A project's tests always run in a single shard, and tests run in parallel within a shard. The balancer only adds up
Expand All @@ -101,18 +116,19 @@ Before the JDBC suite split, the two heaviest projects per pipeline were:
- `:dd-java-agent:instrumentation:jdbc`: 1116.7 s, about 18.6 minutes;
- `:dd-java-agent:instrumentation:lettuce:lettuce-5.0`: 978.1 s, about 16.3 minutes.

After the JDBC and Lettuce splits, their largest independently scheduled units are 321.831 s and 318.271 s. The largest
remaining project is `akka-http-10.0` at 612.3 s. A project's wall-clock span is also longer than its summed test
time: in the first duration-aware pipeline, the unsplit `jdbc` tests (20.5 minutes of summed test time) spanned 27.6
minutes, starting about 10 minutes into the job, so its shard took 39 minutes. Under the previous hash-based
assignment, the `jdbc` shard was already the slowest one.
After the JDBC, Lettuce, Akka HTTP, and Spring WebFlux splits, the largest independently scheduled units from those
projects are 321.831 s, 318.271 s, 166.919 s, and 122.910 s. The largest remaining unsplit weighted project is
`kafka-clients-0.11` at 379.1 s. A project's wall-clock span is also longer than its summed test time: in the first
duration-aware pipeline, the unsplit `jdbc` tests (20.5 minutes of summed test time) spanned 27.6 minutes, starting
about 10 minutes into the job, so its shard took 39 minutes. Under the previous hash-based assignment, the `jdbc`
shard was already the slowest one.

The modeled max/min ratio is optimistic whenever one project approaches the target shard time, so treat the 1.25
bound as a check of the model rather than a guarantee for wall-clock time.

The two heaviest projects must not share a shard, otherwise their floors stack. Capping weights only makes the model
look balanced while letting more work pile onto the affected shard. Lowering the floor requires splitting the work
into independently scheduled units, as the JDBC and Lettuce suite shards do above.
into independently scheduled units, as the JDBC, Lettuce, Akka HTTP, and Spring WebFlux suite shards do above.

## When to refresh

Expand Down
11 changes: 11 additions & 0 deletions settings.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,11 @@ include(
include(
":dd-java-agent:instrumentation:aerospike-4.0",
":dd-java-agent:instrumentation:akka:akka-actor-2.5",
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0:akka-http-base-tests",
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0:akka-http-base-forked-tests",
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0:akka-http-version-101-tests",
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0:akka-http-version-101-forked-tests",
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0:akka-http-version-102-tests",
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.0",
":dd-java-agent:instrumentation:akka:akka-http:akka-http-10.2-iast",
// dd-java-agent:instrumentation:akka:akka-http:akka-http-10.6 will be included when `akkaRepositoryToken` is present, see next `include` block.
Expand Down Expand Up @@ -626,6 +631,12 @@ include(
":dd-java-agent:instrumentation:spring:spring-scheduling-3.1",
":dd-java-agent:instrumentation:spring:spring-security:spring-security-5.0",
":dd-java-agent:instrumentation:spring:spring-security:spring-security-6.0",
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:webflux-client-a-test",
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:webflux-client-b-test",
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:webflux-client-c-test",
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:webflux-client-d-test",
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:webflux-client-other-test",
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0:webflux-forked-iast-tests",
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-5.0",
":dd-java-agent:instrumentation:spring:spring-webflux:spring-webflux-6.0",
":dd-java-agent:instrumentation:spring:spring-webmvc:spring-webmvc-3.1",
Expand Down
Loading