From 4616c9e1383a00042216d8f2b34857a28bf823f6 Mon Sep 17 00:00:00 2001 From: Norbert Orzechowicz Date: Tue, 20 Oct 2026 14:49:43 -0500 Subject: [PATCH] chore(flow-php/flow): speed up and trim CI on pull requests - run extension workflows on PRs only when their code changes - reduce macOS and telemetry matrices on PRs, full matrix on 1.x and nightly - cache compiled PHP extensions and drop redundant apt installs - fix Azurite readiness wait and shorten service health checks - extract shared Rust toolchain action, switch to rust-cache - unify codecov and upload-artifact pins --- .github/actions/codecov-report/action.yml | 10 +- .../rust-extension-toolchain/action.yml | 57 ++++++++++ .github/actions/setup-php-env/action.yml | 18 +++ .github/workflows/baseline.yml | 2 - .github/workflows/job-arrow-extension.yml | 106 +++++------------- .github/workflows/job-flow-php-extension.yml | 105 +++++------------ .github/workflows/job-pg-query-extension.yml | 8 ++ .../workflows/job-phpunit-telemetry-tests.yml | 15 ++- .github/workflows/job-tests.yml | 38 ++----- .github/workflows/release.yml | 1 - .github/workflows/test-arrow-extension.yml | 31 +++++ .github/workflows/test-flow-php-extension.yml | 33 ++++++ .github/workflows/test-pg-query-extension.yml | 28 +++++ .github/workflows/test-suite.yml | 15 +-- .github/workflows/test-website.yml | 1 - 15 files changed, 255 insertions(+), 213 deletions(-) create mode 100644 .github/actions/rust-extension-toolchain/action.yml create mode 100644 .github/workflows/test-arrow-extension.yml create mode 100644 .github/workflows/test-flow-php-extension.yml create mode 100644 .github/workflows/test-pg-query-extension.yml diff --git a/.github/actions/codecov-report/action.yml b/.github/actions/codecov-report/action.yml index 674d177430..9994e7f531 100644 --- a/.github/actions/codecov-report/action.yml +++ b/.github/actions/codecov-report/action.yml @@ -44,7 +44,7 @@ runs: - name: Upload test results (baseline) if: ${{ github.event_name == 'push' && !cancelled() && inputs.php-version == '8.3' && inputs.dependencies == 'locked' }} - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ inputs.token }} directory: ./var/phpunit/logs @@ -52,7 +52,7 @@ runs: - name: Upload code coverage (baseline) if: ${{ github.event_name == 'push' && !cancelled() && inputs.php-version == '8.3' && inputs.dependencies == 'locked' }} - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ inputs.token }} directory: ./var/phpunit/coverage/clover @@ -60,7 +60,7 @@ runs: - name: Upload code coverage (pull request) if: ${{ github.event_name == 'pull_request' && !cancelled() && inputs.dependencies == 'locked' && inputs.php-version == '8.3' }} - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ inputs.token }} directory: ./var/phpunit/coverage/clover @@ -69,7 +69,7 @@ runs: - name: Upload test results (pull request) if: ${{ github.event_name == 'pull_request' && !cancelled() && inputs.dependencies == 'locked' }} - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ inputs.token }} directory: ./var/phpunit/logs @@ -96,7 +96,7 @@ runs: - name: Upload commit SHA as artifact if: ${{ github.event_name == 'push' && inputs.php-version == '8.3' && inputs.dependencies == 'locked' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: codecov-base-commit-sha-${{ steps.baseline.outputs.branch_name }} path: codecov-base-commit-sha-${{ steps.baseline.outputs.branch_name }}.txt diff --git a/.github/actions/rust-extension-toolchain/action.yml b/.github/actions/rust-extension-toolchain/action.yml new file mode 100644 index 0000000000..06de1efb9d --- /dev/null +++ b/.github/actions/rust-extension-toolchain/action.yml @@ -0,0 +1,57 @@ +name: "Rust extension toolchain" +description: "Installs the stable Rust toolchain plus the clang/libclang and autotools that ext-php-rs extensions need to build (directly or through PIE)" + +outputs: + libclang-path: + description: "Directory holding libclang, for LIBCLANG_PATH" + value: ${{ steps.clang.outputs.libclang-path }} + clang-path: + description: "clang executable from the same llvm as libclang, for CLANG_PATH" + value: ${{ steps.clang.outputs.clang-path }} + +runs: + using: "composite" + steps: + - name: "Install Rust toolchain" + uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable + with: + toolchain: stable + + - name: "Install build dependencies (Ubuntu)" + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y build-essential autoconf automake libtool clang libclang-dev + + - name: "Install build dependencies (macOS)" + if: runner.os == 'macOS' + shell: bash + run: brew install autoconf automake libtool llvm + + - name: "Locate clang" + id: clang + shell: bash + run: | + if [ "$RUNNER_OS" = "macOS" ]; then + # Pin the clang executable to the one shipped with the same llvm as + # libclang. Otherwise clang-sys (used by ext-php-rs's bindgen) falls + # back to `xcodebuild -find clang` and mixes Xcode's clang headers + # (e.g. arm_neon.h) with brew's newer libclang, which fails to compile. + echo "libclang-path=$(brew --prefix llvm)/lib" >> "$GITHUB_OUTPUT" + echo "clang-path=$(brew --prefix llvm)/bin/clang" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Find the canonical libclang.so shared library, excluding the + # Clang runtime support libraries under llvm-*/lib/clang/*/lib/. + LIBCLANG_SO=$(find /usr/lib \ + \( -name 'libclang.so' -o -name 'libclang.so.*' -o -name 'libclang-*.so' -o -name 'libclang-*.so.*' \) \ + -not -path '*/clang/*/lib/*' \ + 2>/dev/null | head -1) + if [ -z "$LIBCLANG_SO" ]; then + echo "Could not locate libclang.so under /usr/lib" >&2 + exit 1 + fi + echo "Detected libclang at: $LIBCLANG_SO" + echo "libclang-path=$(dirname "$LIBCLANG_SO")" >> "$GITHUB_OUTPUT" + echo "clang-path=$(command -v clang)" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/setup-php-env/action.yml b/.github/actions/setup-php-env/action.yml index 62ef31c5c0..47c31baee5 100644 --- a/.github/actions/setup-php-env/action.yml +++ b/.github/actions/setup-php-env/action.yml @@ -75,6 +75,24 @@ runs: # Deliberately unquoted: apt-packages is a space-separated list that must word-split. run: sudo apt-get update && sudo apt-get install -y $APT_PACKAGES + - name: "Setup extensions cache" + id: extensions-cache + uses: shivammathur/cache-extensions@22e24453af1bf3851994136f122ce065ab828a74 # 1.15.2 + with: + php-version: ${{ inputs.php-version }} + extensions: "${{ inputs.extensions }}, redis-6.3.0" + key: "php-extensions-${{ runner.os }}-${{ runner.arch }}-v1" + env: + share-cache-across-jobs: true + share-cache-across-workflows: true + + - name: "Cache extensions" + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ steps.extensions-cache.outputs.dir }} + key: ${{ steps.extensions-cache.outputs.key }} + restore-keys: ${{ steps.extensions-cache.outputs.key }} + - name: "Install PHP" uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # 2.40.0 with: diff --git a/.github/workflows/baseline.yml b/.github/workflows/baseline.yml index c6ac836f94..e6e581c6f3 100644 --- a/.github/workflows/baseline.yml +++ b/.github/workflows/baseline.yml @@ -45,7 +45,6 @@ jobs: coverage: "none" extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib' cache-key-suffix: "-locked-phar" - apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev" - name: "Build pg-query-ext from source" uses: "./.github/actions/pg-query-ext" @@ -117,7 +116,6 @@ jobs: dependencies: "locked" extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib' cache-key-suffix: "-website-phar" - apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev" extra-working-directory: "web/landing" extra-cache-key-suffix: "-website" extra-composer-file: "web/landing/composer.lock" diff --git a/.github/workflows/job-arrow-extension.yml b/.github/workflows/job-arrow-extension.yml index f73ad4045f..444a295efe 100644 --- a/.github/workflows/job-arrow-extension.yml +++ b/.github/workflows/job-arrow-extension.yml @@ -2,6 +2,12 @@ name: Arrow Extension on: workflow_call: + inputs: + full-matrix: + description: "Build macOS on every PHP version and test the macOS PIE install; otherwise macOS builds the newest PHP only" + type: boolean + required: false + default: true secrets: CODECOV_TOKEN: required: false @@ -15,6 +21,7 @@ jobs: matrix: os: ['blacksmith-2vcpu-ubuntu-2404', 'macos-latest'] php: ['8.3', '8.4', '8.5'] + exclude: ${{ fromJSON(inputs.full-matrix && '[]' || '[{"os":"macos-latest","php":"8.3"},{"os":"macos-latest","php":"8.4"}]') }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -33,57 +40,23 @@ jobs: coverage: 'pcov' install-dependencies: 'false' - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable - with: - toolchain: stable + - name: Setup Rust extension toolchain + id: rust + uses: ./.github/actions/rust-extension-toolchain - name: Cache cargo registry and build - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - src/extension/arrow-ext/target - key: ${{ runner.os }}-php${{ matrix.php }}-cargo-${{ hashFiles('src/extension/arrow-ext/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-php${{ matrix.php }}-cargo- - - - name: Install build dependencies (Ubuntu) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y build-essential clang libclang-dev - - - name: Install build dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew install llvm - echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> "$GITHUB_ENV" - # Pin the clang executable to the one shipped with the same llvm as - # libclang. Otherwise clang-sys (used by ext-php-rs's bindgen) falls - # back to `xcodebuild -find clang` and mixes Xcode's clang headers - # (e.g. arm_neon.h) with brew's newer libclang, which fails to compile. - echo "CLANG_PATH=$(brew --prefix llvm)/bin/clang" >> "$GITHUB_ENV" - - - name: Set LIBCLANG_PATH (Ubuntu) - if: runner.os == 'Linux' - run: | - # Find the canonical libclang.so shared library, excluding the - # Clang runtime support libraries under llvm-*/lib/clang/*/lib/. - LIBCLANG_SO=$(find /usr/lib \ - \( -name 'libclang.so' -o -name 'libclang.so.*' -o -name 'libclang-*.so' -o -name 'libclang-*.so.*' \) \ - -not -path '*/clang/*/lib/*' \ - 2>/dev/null | head -1) - if [ -z "$LIBCLANG_SO" ]; then - echo "Could not locate libclang.so under /usr/lib" >&2 - exit 1 - fi - echo "LIBCLANG_PATH=$(dirname "$LIBCLANG_SO")" >> "$GITHUB_ENV" - echo "Detected libclang at: $LIBCLANG_SO" + # ext-php-rs bindings are generated against the PHP headers, so builds are per PHP version + key: php${{ matrix.php }} + workspaces: | + src/extension/arrow-ext -> target - name: Build extension working-directory: src/extension/arrow-ext + env: &rust-env + LIBCLANG_PATH: ${{ steps.rust.outputs.libclang-path }} + CLANG_PATH: ${{ steps.rust.outputs.clang-path }} run: make build - name: Install Composer Dependencies @@ -94,6 +67,7 @@ jobs: - name: Run PHPT tests working-directory: src/extension/arrow-ext + env: *rust-env run: make test - name: Install extension and verify @@ -124,6 +98,7 @@ jobs: fail-fast: false matrix: os: ['blacksmith-2vcpu-ubuntu-2404', 'macos-latest'] + exclude: ${{ fromJSON(inputs.full-matrix && '[]' || '[{"os":"macos-latest"}]') }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -140,41 +115,9 @@ jobs: extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib' tools: 'composer:v2, phpize, php-config' - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable - with: - toolchain: stable - - - name: Install build dependencies (Ubuntu) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y build-essential autoconf automake libtool clang libclang-dev - - - name: Install build dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew install autoconf automake libtool llvm - echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> "$GITHUB_ENV" - # See the build job: keep clang and libclang from the same llvm so - # bindgen does not mix brew's libclang with Xcode's clang headers. - echo "CLANG_PATH=$(brew --prefix llvm)/bin/clang" >> "$GITHUB_ENV" - - - name: Set LIBCLANG_PATH (Ubuntu) - if: runner.os == 'Linux' - run: | - # Find the canonical libclang.so shared library, excluding the - # Clang runtime support libraries under llvm-*/lib/clang/*/lib/. - LIBCLANG_SO=$(find /usr/lib \ - \( -name 'libclang.so' -o -name 'libclang.so.*' -o -name 'libclang-*.so' -o -name 'libclang-*.so.*' \) \ - -not -path '*/clang/*/lib/*' \ - 2>/dev/null | head -1) - if [ -z "$LIBCLANG_SO" ]; then - echo "Could not locate libclang.so under /usr/lib" >&2 - exit 1 - fi - echo "LIBCLANG_PATH=$(dirname "$LIBCLANG_SO")" >> "$GITHUB_ENV" - echo "Detected libclang at: $LIBCLANG_SO" + - name: Setup Rust extension toolchain + id: rust + uses: ./.github/actions/rust-extension-toolchain - name: Install PIE run: | @@ -191,6 +134,9 @@ jobs: mv composer.tmp.json composer.json - name: Install extension via PIE (from local) + env: + LIBCLANG_PATH: ${{ steps.rust.outputs.libclang-path }} + CLANG_PATH: ${{ steps.rust.outputs.clang-path }} run: | sudo pie repository:remove packagist.org sudo pie repository:add path "$RUNNER_TEMP/src/extension/arrow-ext" diff --git a/.github/workflows/job-flow-php-extension.yml b/.github/workflows/job-flow-php-extension.yml index 43ac6c628b..5e4508f1e3 100644 --- a/.github/workflows/job-flow-php-extension.yml +++ b/.github/workflows/job-flow-php-extension.yml @@ -2,6 +2,12 @@ name: Flow PHP Extension on: workflow_call: + inputs: + full-matrix: + description: "Build macOS on every PHP version and test the macOS PIE install; otherwise macOS builds the newest PHP only" + type: boolean + required: false + default: true secrets: CODECOV_TOKEN: required: false @@ -15,6 +21,7 @@ jobs: matrix: os: ['blacksmith-2vcpu-ubuntu-2404', 'macos-latest'] php: ['8.3', '8.4', '8.5'] + exclude: ${{ fromJSON(inputs.full-matrix && '[]' || '[{"os":"macos-latest","php":"8.3"},{"os":"macos-latest","php":"8.4"}]') }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -33,60 +40,35 @@ jobs: coverage: 'pcov' install-dependencies: 'false' - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable - with: - toolchain: stable + - name: Setup Rust extension toolchain + id: rust + uses: ./.github/actions/rust-extension-toolchain - name: Cache cargo registry and build - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - src/extension/flow-php-ext/target - src/extension/arrow-ext/target - key: ${{ runner.os }}-php${{ matrix.php }}-cargo-${{ hashFiles('src/extension/flow-php-ext/Cargo.lock', 'src/extension/arrow-ext/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-php${{ matrix.php }}-cargo- - - - name: Install build dependencies (Ubuntu) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y build-essential clang libclang-dev - - - name: Install build dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew install llvm - echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> "$GITHUB_ENV" - echo "CLANG_PATH=$(brew --prefix llvm)/bin/clang" >> "$GITHUB_ENV" - - - name: Set LIBCLANG_PATH (Ubuntu) - if: runner.os == 'Linux' - run: | - LIBCLANG_SO=$(find /usr/lib \ - \( -name 'libclang.so' -o -name 'libclang.so.*' -o -name 'libclang-*.so' -o -name 'libclang-*.so.*' \) \ - -not -path '*/clang/*/lib/*' \ - 2>/dev/null | head -1) - if [ -z "$LIBCLANG_SO" ]; then - echo "Could not locate libclang.so under /usr/lib" >&2 - exit 1 - fi - echo "LIBCLANG_PATH=$(dirname "$LIBCLANG_SO")" >> "$GITHUB_ENV" - echo "Detected libclang at: $LIBCLANG_SO" + # ext-php-rs bindings are generated against the PHP headers, so builds are per PHP version + key: php${{ matrix.php }} + workspaces: | + src/extension/flow-php-ext -> target + src/extension/flow-php-ext/crates/flow-batch-frame -> target + src/extension/arrow-ext -> target - name: Build arrow extension working-directory: src/extension/arrow-ext + env: &rust-env + LIBCLANG_PATH: ${{ steps.rust.outputs.libclang-path }} + CLANG_PATH: ${{ steps.rust.outputs.clang-path }} run: make build - name: Build extension working-directory: src/extension/flow-php-ext + env: *rust-env run: make build - name: Test flow-batch-frame crate working-directory: src/extension/flow-php-ext/crates/flow-batch-frame + env: *rust-env run: cargo test --locked - name: Install Composer Dependencies @@ -97,6 +79,7 @@ jobs: - name: Run PHPT tests working-directory: src/extension/flow-php-ext + env: *rust-env run: make test ARROW_SO=../arrow-ext/ext/modules/arrow.so - name: Install extension and verify @@ -136,6 +119,7 @@ jobs: fail-fast: false matrix: os: ['blacksmith-2vcpu-ubuntu-2404', 'macos-latest'] + exclude: ${{ fromJSON(inputs.full-matrix && '[]' || '[{"os":"macos-latest"}]') }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -152,41 +136,9 @@ jobs: extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib' tools: 'composer:v2, phpize, php-config' - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable - with: - toolchain: stable - - - name: Install build dependencies (Ubuntu) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y build-essential autoconf automake libtool clang libclang-dev - - - name: Install build dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew install autoconf automake libtool llvm - echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> "$GITHUB_ENV" - # See the build job: keep clang and libclang from the same llvm so - # bindgen does not mix brew's libclang with Xcode's clang headers. - echo "CLANG_PATH=$(brew --prefix llvm)/bin/clang" >> "$GITHUB_ENV" - - - name: Set LIBCLANG_PATH (Ubuntu) - if: runner.os == 'Linux' - run: | - # Find the canonical libclang.so shared library, excluding the - # Clang runtime support libraries under llvm-*/lib/clang/*/lib/. - LIBCLANG_SO=$(find /usr/lib \ - \( -name 'libclang.so' -o -name 'libclang.so.*' -o -name 'libclang-*.so' -o -name 'libclang-*.so.*' \) \ - -not -path '*/clang/*/lib/*' \ - 2>/dev/null | head -1) - if [ -z "$LIBCLANG_SO" ]; then - echo "Could not locate libclang.so under /usr/lib" >&2 - exit 1 - fi - echo "LIBCLANG_PATH=$(dirname "$LIBCLANG_SO")" >> "$GITHUB_ENV" - echo "Detected libclang at: $LIBCLANG_SO" + - name: Setup Rust extension toolchain + id: rust + uses: ./.github/actions/rust-extension-toolchain - name: Install PIE run: | @@ -203,6 +155,9 @@ jobs: mv composer.tmp.json composer.json - name: Install extension via PIE (from local) + env: + LIBCLANG_PATH: ${{ steps.rust.outputs.libclang-path }} + CLANG_PATH: ${{ steps.rust.outputs.clang-path }} run: | sudo pie repository:remove packagist.org sudo pie repository:add path "$RUNNER_TEMP/src/extension/flow-php-ext" diff --git a/.github/workflows/job-pg-query-extension.yml b/.github/workflows/job-pg-query-extension.yml index 355f4d2208..012aa97a96 100644 --- a/.github/workflows/job-pg-query-extension.yml +++ b/.github/workflows/job-pg-query-extension.yml @@ -2,6 +2,12 @@ name: PG Query Extension on: workflow_call: + inputs: + full-matrix: + description: "Build macOS on every PHP version and test the macOS PIE install; otherwise macOS builds the newest PHP only" + type: boolean + required: false + default: true jobs: build: @@ -12,6 +18,7 @@ jobs: matrix: os: ['blacksmith-2vcpu-ubuntu-2404', 'macos-latest'] php: ['8.3', '8.4', '8.5'] + exclude: ${{ fromJSON(inputs.full-matrix && '[]' || '[{"os":"macos-latest","php":"8.3"},{"os":"macos-latest","php":"8.4"}]') }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -65,6 +72,7 @@ jobs: fail-fast: false matrix: os: ['blacksmith-2vcpu-ubuntu-2404', 'macos-latest'] + exclude: ${{ fromJSON(inputs.full-matrix && '[]' || '[{"os":"macos-latest"}]') }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/job-phpunit-telemetry-tests.yml b/.github/workflows/job-phpunit-telemetry-tests.yml index a98e3da236..b3ae9fe2ea 100644 --- a/.github/workflows/job-phpunit-telemetry-tests.yml +++ b/.github/workflows/job-phpunit-telemetry-tests.yml @@ -2,6 +2,12 @@ name: PHPUnit Telemetry Bridge Tests on: workflow_call: + inputs: + full-matrix: + description: "Every PHPUnit x PHP combination; otherwise one PHP version per PHPUnit major" + type: boolean + required: false + default: true secrets: CODECOV_TOKEN: required: false @@ -22,7 +28,7 @@ jobs: POSTGRES_DB: postgres ports: - 5432/tcp - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + options: --health-cmd pg_isready --health-interval 2s --health-timeout 5s --health-retries 25 strategy: fail-fast: false matrix: @@ -34,10 +40,8 @@ jobs: - "8.3" - "8.4" - "8.5" - exclude: - # PHPUnit 13 requires PHP >= 8.4.1. - - phpunit-version: "13" - php-version: "8.3" + # PHPUnit 13 requires PHP >= 8.4.1, so 13 x 8.3 is always excluded. + exclude: ${{ fromJSON(inputs.full-matrix && '[{"phpunit-version":"13","php-version":"8.3"}]' || '[{"phpunit-version":"13","php-version":"8.3"},{"phpunit-version":"13","php-version":"8.4"},{"phpunit-version":"12","php-version":"8.3"},{"phpunit-version":"12","php-version":"8.5"},{"phpunit-version":"11","php-version":"8.4"},{"phpunit-version":"11","php-version":"8.5"}]') }} steps: - name: "Checkout" @@ -53,7 +57,6 @@ jobs: coverage: ${{ matrix.php-version == '8.3' && 'pcov' || 'none' }} extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, curl, pgsql, grpc, protobuf' ini-values: 'memory_limit=-1' - apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev" - name: "Build pg-query-ext from source" uses: "./.github/actions/pg-query-ext" diff --git a/.github/workflows/job-tests.yml b/.github/workflows/job-tests.yml index 4c20496a2b..d3f27cca71 100644 --- a/.github/workflows/job-tests.yml +++ b/.github/workflows/job-tests.yml @@ -22,7 +22,7 @@ jobs: POSTGRES_DB: postgres ports: - 5432/tcp - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + options: --health-cmd pg_isready --health-interval 2s --health-timeout 5s --health-retries 25 mysql: image: mysql:8.0.31-debian ports: @@ -40,9 +40,9 @@ jobs: ES_JAVA_OPTS: "-Xms512m -Xmx512m" options: >- --health-cmd "curl http://localhost:9200/_cluster/health" - --health-interval 10s + --health-interval 2s --health-timeout 5s - --health-retries 10 + --health-retries 50 ports: - 9200/tcp strategy: @@ -112,7 +112,6 @@ jobs: coverage: ${{ (matrix.php-version == '8.3' && matrix.dependencies == 'locked') && 'pcov' || 'none' }} extensions: ':psr, apcu, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, curl, pgsql, grpc, protobuf' ini-values: 'memory_limit=-1, post_max_size=32M, upload_max_filesize=32M, apc.enable_cli=1' - apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev" - name: "Build pg-query-ext from source" uses: "./.github/actions/pg-query-ext" @@ -131,15 +130,9 @@ jobs: shell: bash run: | azurite-blob --blobHost 0.0.0.0 --blobPort 10000 > azurite.log 2>&1 & - for i in {1..30}; do - if curl -f http://localhost:10000/ 2>/dev/null; then - echo "Azurite is ready" - break - fi - echo "Waiting for Azurite... ($i/30)" - sleep 1 - done - cat azurite.log + # any HTTP response means the endpoint is up; Azurite answers the bare root with an error status + timeout 30 sh -c 'until curl -s -o /dev/null http://localhost:10000/; do sleep 1; done' \ + || { echo "Azurite did not become ready"; cat azurite.log; exit 1; } env: AZURITE_ACCOUNTS: flowphpaccount01:flowphpkey01 @@ -148,7 +141,7 @@ jobs: - name: "Test" timeout-minutes: 15 run: "just test --exclude-group grpc --log-junit ./var/phpunit/logs/junit.xml ${{ (matrix.php-version == '8.3' && matrix.dependencies == 'locked') && '--coverage-clover=./var/phpunit/coverage/clover/coverage.xml' || '' }}" - env: + env: &test-env PGSQL_DATABASE_URL: pgsql://postgres:postgres@127.0.0.1:${{ job.services.postgres.ports['5432'] }}/postgres?serverVersion=18&charset=utf8 MYSQL_DATABASE_URL: mysql://mysql:mysql@127.0.0.1:${{ job.services.mysql.ports['3306'] }}/mysql ELASTICSEARCH_URL: http://127.0.0.1:${{ job.services.elasticsearch.ports['9200'] }} @@ -178,22 +171,7 @@ jobs: timeout-minutes: 10 continue-on-error: ${{ matrix.php-version == '8.5' }} run: "just test --group grpc --log-junit ./var/phpunit/logs/grpc-junit.xml ${{ (matrix.php-version == '8.3' && matrix.dependencies == 'locked') && '--coverage-clover=./var/phpunit/coverage/clover/grpc-coverage.xml' || '' }}" - env: - PGSQL_DATABASE_URL: pgsql://postgres:postgres@127.0.0.1:${{ job.services.postgres.ports['5432'] }}/postgres?serverVersion=18&charset=utf8 - MYSQL_DATABASE_URL: mysql://mysql:mysql@127.0.0.1:${{ job.services.mysql.ports['3306'] }}/mysql - SQLITE_DATABASE_URL: "sqlite:///:memory:" - AZURITE_HOST: "localhost" - AZURITE_BLOB_PORT: "10000" - AZURITE_ACCOUNT_NAME: "flowphpaccount01" - AZURITE_ACCOUNT_KEY: "flowphpkey01" - S3_ENDPOINT: "http://localhost:9000" - S3_REGION: "us-east-1" - S3_ACCESS_KEY_ID: "flowphpaccess01" - S3_SECRET_ACCESS_KEY: "flowphpsecret01" - OTEL_RECEIVER_HTTP_ENDPOINT: "http://localhost:4318" - OTEL_RECEIVER_GRPC_ENDPOINT: "localhost:4317" - OTEL_COLLECTOR_METRICS_ENDPOINT: "http://localhost:8888/metrics" - FLOW_PARQUET_TESTS_DEBUG: "0" + env: *test-env - name: Upload to Codecov uses: ./.github/actions/codecov-report diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39c04a00d7..81053f523f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,7 +34,6 @@ jobs: coverage: "none" extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib' cache-key-suffix: "-locked-phar" - apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev" - name: "Build pg-query-ext from source" uses: "./.github/actions/pg-query-ext" diff --git a/.github/workflows/test-arrow-extension.yml b/.github/workflows/test-arrow-extension.yml new file mode 100644 index 0000000000..d425c9e7cc --- /dev/null +++ b/.github/workflows/test-arrow-extension.yml @@ -0,0 +1,31 @@ +name: Arrow Extension + +on: + pull_request: + paths: + - 'src/extension/arrow-ext/**' + - '.github/workflows/test-arrow-extension.yml' + - '.github/workflows/job-arrow-extension.yml' + - '.github/actions/setup-php-env/**' + - '.github/actions/composer-install/**' + - '.github/actions/rust-extension-toolchain/**' + push: + branches: [ 1.x ] + schedule: + - cron: '0 4 * * *' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + arrow-extension: + uses: ./.github/workflows/job-arrow-extension.yml + with: + full-matrix: ${{ github.event_name != 'pull_request' }} + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/test-flow-php-extension.yml b/.github/workflows/test-flow-php-extension.yml new file mode 100644 index 0000000000..9957bd83d8 --- /dev/null +++ b/.github/workflows/test-flow-php-extension.yml @@ -0,0 +1,33 @@ +name: Flow PHP Extension + +on: + pull_request: + paths: + - 'src/extension/flow-php-ext/**' + # the flow-php-ext build and PHPT tests load arrow-ext + - 'src/extension/arrow-ext/**' + - '.github/workflows/test-flow-php-extension.yml' + - '.github/workflows/job-flow-php-extension.yml' + - '.github/actions/setup-php-env/**' + - '.github/actions/composer-install/**' + - '.github/actions/rust-extension-toolchain/**' + push: + branches: [ 1.x ] + schedule: + - cron: '0 4 * * *' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + flow-php-extension: + uses: ./.github/workflows/job-flow-php-extension.yml + with: + full-matrix: ${{ github.event_name != 'pull_request' }} + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/test-pg-query-extension.yml b/.github/workflows/test-pg-query-extension.yml new file mode 100644 index 0000000000..2db892aa83 --- /dev/null +++ b/.github/workflows/test-pg-query-extension.yml @@ -0,0 +1,28 @@ +name: PG Query Extension + +on: + pull_request: + paths: + - 'src/extension/pg-query-ext/**' + - '.github/workflows/test-pg-query-extension.yml' + - '.github/workflows/job-pg-query-extension.yml' + - '.github/actions/setup-php-env/**' + - '.github/actions/composer-install/**' + push: + branches: [ 1.x ] + schedule: + - cron: '0 4 * * *' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + pg-query-extension: + uses: ./.github/workflows/job-pg-query-extension.yml + with: + full-matrix: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 56167f8d6a..9d8a3bcf1d 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -48,6 +48,8 @@ jobs: phpunit-telemetry-tests: uses: ./.github/workflows/job-phpunit-telemetry-tests.yml + with: + full-matrix: ${{ github.event_name != 'pull_request' }} secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} @@ -56,18 +58,5 @@ jobs: secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - pg-query-extension: - uses: ./.github/workflows/job-pg-query-extension.yml - - arrow-extension: - uses: ./.github/workflows/job-arrow-extension.yml - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - flow-php-extension: - uses: ./.github/workflows/job-flow-php-extension.yml - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - mutation-tests: uses: ./.github/workflows/job-mutation-tests.yml diff --git a/.github/workflows/test-website.yml b/.github/workflows/test-website.yml index 99fdaba8b3..499c3db13c 100644 --- a/.github/workflows/test-website.yml +++ b/.github/workflows/test-website.yml @@ -37,7 +37,6 @@ jobs: coverage: "none" extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib' cache-key-suffix: "-locked-phar" - apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev" - name: "Build pg-query-ext from source" uses: "./.github/actions/pg-query-ext"