From 537c006e9497713156c601c80d3c5811f36abbe7 Mon Sep 17 00:00:00 2001 From: Prabhash Date: Fri, 25 Sep 2026 13:45:15 +0100 Subject: [PATCH 1/2] ci: shard Go tests and parallelize CDP suite --- .github/workflows/ci.yml | 45 +++++++++++++++++++++++---- pkg/driver/browser/cdp/driver_test.go | 4 +++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6999921..84e8b97d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,9 +10,28 @@ on: workflow_dispatch: jobs: - test: - name: Test + test-go: + name: Test Go (shard ${{ matrix.shard.id }}) runs-on: ubuntu-latest + env: + CDP_PARALLEL: "4" + strategy: + fail-fast: false + # Static sharding avoids a timing/preparation run. The slow CDP browser + # package runs alone in its own shard; the remaining packages are split + # across three shards. Keep node_modules out of these lists — they are + # not repository test packages. Update the lists when packages are added + # or renamed. + matrix: + shard: + - id: cdp + packages: "github.com/devicelab-dev/maestro-runner/pkg/driver/browser/cdp" + - id: s1 + packages: "github.com/devicelab-dev/maestro-runner/pkg/cli github.com/devicelab-dev/maestro-runner/pkg/core github.com/devicelab-dev/maestro-runner/pkg/driver/devicelab github.com/devicelab-dev/maestro-runner/pkg/driver/uiautomator2 github.com/devicelab-dev/maestro-runner/pkg/executor github.com/devicelab-dev/maestro-runner/pkg/jsengine github.com/devicelab-dev/maestro-runner/pkg/report github.com/devicelab-dev/maestro-runner/pkg/uiautomator2" + - id: s2 + packages: "github.com/devicelab-dev/maestro-runner github.com/devicelab-dev/maestro-runner/pkg/cloud github.com/devicelab-dev/maestro-runner/pkg/device github.com/devicelab-dev/maestro-runner/pkg/driver/devicelab_ios github.com/devicelab-dev/maestro-runner/pkg/driver/wda github.com/devicelab-dev/maestro-runner/pkg/flow github.com/devicelab-dev/maestro-runner/pkg/logger github.com/devicelab-dev/maestro-runner/pkg/validator" + - id: s3 + packages: "github.com/devicelab-dev/maestro-runner/drivers github.com/devicelab-dev/maestro-runner/pkg/config github.com/devicelab-dev/maestro-runner/pkg/driver/appium github.com/devicelab-dev/maestro-runner/pkg/driver/mock github.com/devicelab-dev/maestro-runner/pkg/emulator github.com/devicelab-dev/maestro-runner/pkg/flutter github.com/devicelab-dev/maestro-runner/pkg/maestro github.com/devicelab-dev/maestro-runner/pkg/simulator" steps: - name: Checkout code uses: actions/checkout@v5 @@ -26,13 +45,27 @@ jobs: - name: Download dependencies run: go mod download - - name: Run tests - run: go test -race -timeout 25m -coverprofile=coverage.out -covermode=atomic ./... + - name: Run tests (shard ${{ matrix.shard.id }}) + env: + # Race detection makes the browser-backed CDP tests substantially + # slower. The other shards retain -race; CDP is bounded separately. + RACE_FLAG: ${{ matrix.shard.id != 'cdp' && '-race' || '' }} + run: | + PKGS="${{ matrix.shard.packages }}" + echo "Testing shard ${{ matrix.shard.id }} (race='$RACE_FLAG'):" + echo "$PKGS" + # shellcheck disable=SC2086 + go test $RACE_FLAG -parallel "$CDP_PARALLEL" -timeout 25m -v -coverprofile=coverage-${{ matrix.shard.id }}.out -covermode=atomic $PKGS - name: Upload coverage + if: ${{ hashFiles(format('coverage-{0}.out', matrix.shard.id)) != '' }} uses: codecov/codecov-action@v5 + # The shared flag and name let Codecov combine each shard's partial + # upload into one coverage report for the commit. with: - files: ./coverage.out + files: ./coverage-${{ matrix.shard.id }}.out + flags: unittests + name: coverage-report token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false @@ -65,7 +98,7 @@ jobs: build: name: Build runs-on: ubuntu-latest - needs: [test, lint] + needs: [test-go, lint] steps: - name: Checkout code uses: actions/checkout@v5 diff --git a/pkg/driver/browser/cdp/driver_test.go b/pkg/driver/browser/cdp/driver_test.go index 8b5fc8e1..28bf2690 100644 --- a/pkg/driver/browser/cdp/driver_test.go +++ b/pkg/driver/browser/cdp/driver_test.go @@ -146,6 +146,10 @@ func newTestServer() *httptest.Server { // newTestDriver creates a Driver connected to the test server. func newTestDriver(t *testing.T, serverURL string) *Driver { t.Helper() + // Each test creates its own server and headless Chromium instance, so the + // browser-backed tests are independent and can run concurrently. The + // workflow's -parallel flag bounds the number of active browser instances. + t.Parallel() d, err := New(Config{ Headless: true, URL: serverURL, From e29f5958648b6dfccbc4afdfb9845778a82dba4b Mon Sep 17 00:00:00 2001 From: Prabhash Date: Tue, 6 Oct 2026 12:12:55 +0530 Subject: [PATCH 2/2] ci: guard the static test shards against package drift The shard package lists in ci.yml are static, so a Go package added later would never be tested and CI would still be green. Add a step that diffs 'go list ./...' against the shard lists. The lists are read back out of this workflow file rather than repeated in the check, because a second copy would drift and then validate itself instead of the matrix - missing a typo or a package dropped from a shard. The check also catches a package assigned to two shards, since the sort is deliberately not 'sort -u'. Also rename CDP_PARALLEL to TEST_PARALLEL: 'go test -parallel' is a package-wide flag applied to every shard, so the old name implied it only affected CDP. --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84e8b97d..d8a973d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: name: Test Go (shard ${{ matrix.shard.id }}) runs-on: ubuntu-latest env: - CDP_PARALLEL: "4" + TEST_PARALLEL: "4" strategy: fail-fast: false # Static sharding avoids a timing/preparation run. The slow CDP browser @@ -45,6 +45,29 @@ jobs: - name: Download dependencies run: go mod download + - name: Verify every Go package is in a shard + # Reads the shard lists back out of this file instead of repeating them: + # a second copy would drift and then validate itself, hiding a typo or a + # dropped package. `sort` is not `sort -u`, so a package listed in two + # shards still shows up. A package that cannot build on this runner + # (e.g. //go:build darwin) is absent from `go list`, so listing it fails + # — and omitting it hides it from this check entirely. + run: | + set -euo pipefail + workdir=$(mktemp -d) + trap 'rm -rf "$workdir"' EXIT + go list ./... | sort > "$workdir/all-packages.txt" + grep -oE '^[[:space:]]+packages: .*$' .github/workflows/ci.yml \ + | sed -e "s/^[[:space:]]*packages: ['\"]//" -e "s/['\"]$//" \ + | tr ' ' '\n' \ + | grep -v '^$' \ + | sort > "$workdir/sharded-packages.txt" + if ! diff -u "$workdir/all-packages.txt" "$workdir/sharded-packages.txt"; then + echo "::error::Go packages and test shards are out of sync. Every package from 'go list ./...' must appear in exactly one shard." + exit 1 + fi + echo "All $(wc -l < "$workdir/all-packages.txt" | tr -d ' ') Go packages are covered by a test shard." + - name: Run tests (shard ${{ matrix.shard.id }}) env: # Race detection makes the browser-backed CDP tests substantially @@ -55,7 +78,7 @@ jobs: echo "Testing shard ${{ matrix.shard.id }} (race='$RACE_FLAG'):" echo "$PKGS" # shellcheck disable=SC2086 - go test $RACE_FLAG -parallel "$CDP_PARALLEL" -timeout 25m -v -coverprofile=coverage-${{ matrix.shard.id }}.out -covermode=atomic $PKGS + go test $RACE_FLAG -parallel "$TEST_PARALLEL" -timeout 25m -v -coverprofile=coverage-${{ matrix.shard.id }}.out -covermode=atomic $PKGS - name: Upload coverage if: ${{ hashFiles(format('coverage-{0}.out', matrix.shard.id)) != '' }}