From 31c591f8ab4c39f7472019668f808b39ca7de5f8 Mon Sep 17 00:00:00 2001 From: Charles Cote Date: Thu, 8 Oct 2026 12:02:16 +0200 Subject: [PATCH 1/6] ci: test fuller system test shards --- .github/workflows/run-system-tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-system-tests.yaml b/.github/workflows/run-system-tests.yaml index 4b0ad663659..a96cd540e0b 100644 --- a/.github/workflows/run-system-tests.yaml +++ b/.github/workflows/run-system-tests.yaml @@ -72,7 +72,8 @@ jobs: # If you change the following comment, update the pattern in the update_system_test_reference.sh script to match. ref: "main" # system tests are pinned on release branches only binaries_artifact: binaries - desired_execution_time: 900 # 15 minutes + # Experiment with fewer, fuller shards to reduce repeated environment setup costs. + desired_execution_time: 1200 # 20 minutes scenarios_groups: tracer-release excluded_scenarios: APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,PROFILING # exclude flaky scenarios skip_empty_scenarios: ${{ github.event_name != 'push' && github.event_name != 'schedule' }} From 9eb8cf6fe50485ca2cdf4ca4b2cb85de57556e00 Mon Sep 17 00:00:00 2001 From: Charles Cote Date: Thu, 8 Oct 2026 13:41:33 +0200 Subject: [PATCH 2/6] ci: test smaller system test shards --- .github/workflows/run-system-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-system-tests.yaml b/.github/workflows/run-system-tests.yaml index a96cd540e0b..e98189f763c 100644 --- a/.github/workflows/run-system-tests.yaml +++ b/.github/workflows/run-system-tests.yaml @@ -72,8 +72,8 @@ jobs: # If you change the following comment, update the pattern in the update_system_test_reference.sh script to match. ref: "main" # system tests are pinned on release branches only binaries_artifact: binaries - # Experiment with fewer, fuller shards to reduce repeated environment setup costs. - desired_execution_time: 1200 # 20 minutes + # Experiment with smaller shards to reduce the slow tail and overall workflow duration. + desired_execution_time: 750 # 12.5 minutes scenarios_groups: tracer-release excluded_scenarios: APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,PROFILING # exclude flaky scenarios skip_empty_scenarios: ${{ github.event_name != 'push' && github.event_name != 'schedule' }} From 5ad084d0a1e1e04896123ff758dc2d9bb8b1b3de Mon Sep 17 00:00:00 2001 From: Charles Cote Date: Thu, 8 Oct 2026 14:13:43 +0200 Subject: [PATCH 3/6] ci: test maximum system test parallelism --- .github/workflows/run-system-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-system-tests.yaml b/.github/workflows/run-system-tests.yaml index e98189f763c..a2ac291f634 100644 --- a/.github/workflows/run-system-tests.yaml +++ b/.github/workflows/run-system-tests.yaml @@ -73,7 +73,7 @@ jobs: ref: "main" # system tests are pinned on release branches only binaries_artifact: binaries # Experiment with smaller shards to reduce the slow tail and overall workflow duration. - desired_execution_time: 750 # 12.5 minutes + desired_execution_time: 720 # 12 minutes scenarios_groups: tracer-release excluded_scenarios: APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,PROFILING # exclude flaky scenarios skip_empty_scenarios: ${{ github.event_name != 'push' && github.event_name != 'schedule' }} From 8e1948fa52894ec3517ef85b05b669ce74ccc29c Mon Sep 17 00:00:00 2001 From: Charles Cote Date: Thu, 8 Oct 2026 14:49:55 +0200 Subject: [PATCH 4/6] Keep the fastest system test shard target --- .github/workflows/run-system-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-system-tests.yaml b/.github/workflows/run-system-tests.yaml index a2ac291f634..e98189f763c 100644 --- a/.github/workflows/run-system-tests.yaml +++ b/.github/workflows/run-system-tests.yaml @@ -73,7 +73,7 @@ jobs: ref: "main" # system tests are pinned on release branches only binaries_artifact: binaries # Experiment with smaller shards to reduce the slow tail and overall workflow duration. - desired_execution_time: 720 # 12 minutes + desired_execution_time: 750 # 12.5 minutes scenarios_groups: tracer-release excluded_scenarios: APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,PROFILING # exclude flaky scenarios skip_empty_scenarios: ${{ github.event_name != 'push' && github.event_name != 'schedule' }} From 4c2d5974ce4f63720de7b8035d852dc663580d3f Mon Sep 17 00:00:00 2001 From: Charles Cote Date: Thu, 8 Oct 2026 15:23:48 +0200 Subject: [PATCH 5/6] Probe a thirteen minute system test shard target --- .github/workflows/run-system-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-system-tests.yaml b/.github/workflows/run-system-tests.yaml index e98189f763c..9bdd2073eaf 100644 --- a/.github/workflows/run-system-tests.yaml +++ b/.github/workflows/run-system-tests.yaml @@ -73,7 +73,7 @@ jobs: ref: "main" # system tests are pinned on release branches only binaries_artifact: binaries # Experiment with smaller shards to reduce the slow tail and overall workflow duration. - desired_execution_time: 750 # 12.5 minutes + desired_execution_time: 780 # 13 minutes scenarios_groups: tracer-release excluded_scenarios: APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,PROFILING # exclude flaky scenarios skip_empty_scenarios: ${{ github.event_name != 'push' && github.event_name != 'schedule' }} From 1f529938c942de20aa33969873178ee57e3dd5c3 Mon Sep 17 00:00:00 2001 From: Charles Cote Date: Thu, 8 Oct 2026 16:00:07 +0200 Subject: [PATCH 6/6] Select the fastest system test shard target --- .github/workflows/run-system-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-system-tests.yaml b/.github/workflows/run-system-tests.yaml index 9bdd2073eaf..bc6e809ded1 100644 --- a/.github/workflows/run-system-tests.yaml +++ b/.github/workflows/run-system-tests.yaml @@ -72,8 +72,8 @@ jobs: # If you change the following comment, update the pattern in the update_system_test_reference.sh script to match. ref: "main" # system tests are pinned on release branches only binaries_artifact: binaries - # Experiment with smaller shards to reduce the slow tail and overall workflow duration. - desired_execution_time: 780 # 13 minutes + # Use historical durations to reduce the slow tail without hitting the 256-job cap. + desired_execution_time: 750 # 12.5 minutes scenarios_groups: tracer-release excluded_scenarios: APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,PROFILING # exclude flaky scenarios skip_empty_scenarios: ${{ github.event_name != 'push' && github.event_name != 'schedule' }}