diff --git a/.github/workflows/mvn-ci-build.yml b/.github/workflows/mvn-ci-build.yml index 92e478940..401148331 100644 --- a/.github/workflows/mvn-ci-build.yml +++ b/.github/workflows/mvn-ci-build.yml @@ -29,19 +29,125 @@ on: branches: - "main" +# The build cache extension does not hash the JDK, so it is part of every cache +# key below. Defined once here rather than repeated, so that changing the JDK +# cannot leave a key asserting the old one. +env: + JDK_VERSION: '11' + jobs: build: runs-on: ubuntu-latest - timeout-minutes: 30 + # A cold build is ~27 min, which left no headroom under the previous 30. + timeout-minutes: 40 steps: - uses: actions/checkout@v7 - - name: Set up JDK 11 + - name: Set up JDK ${{ env.JDK_VERSION }} uses: actions/setup-java@v5 with: - java-version: '11' + java-version: ${{ env.JDK_VERSION }} distribution: 'temurin' - cache: maven + + # Restore only. License Check writes this cache, and only on `main`: it + # finishes in a couple of minutes against this job's 27, and it stops at + # `package` so the entry never carries this project's artifacts. It + # resolves the reactor explicitly for this job's benefit; its own license + # steps reach only part of it. + # Previously both jobs used setup-java's `cache: maven`, which gave them + # the same key, and whichever finished first reserved it - this job then + # logged "Unable to reserve cache with key" on every single run. Making + # the reader explicit removes the race rather than winning it. + # + # A warm start, not a guarantee: `install` here runs plugins that a build + # stopping at `package` never resolves, so expect a delta every run. + - name: Restore Maven repository cache + uses: actions/cache/restore@v6 + with: + path: ~/.m2/repository + key: m2-repository-${{ runner.os }}-jdk${{ env.JDK_VERSION }}-${{ hashFiles('**/pom.xml', '**/.mvn/extensions.xml') }} + restore-keys: | + m2-repository-${{ runner.os }}-jdk${{ env.JDK_VERSION }}- + + - name: Restore Maven wrapper cache + uses: actions/cache/restore@v6 + with: + path: ~/.m2/wrapper/dists + key: m2-wrapper-${{ runner.os }}-${{ hashFiles('**/.mvn/wrapper/maven-wrapper.properties') }} + + # A separate cache from the repository above: the build cache extension + # configured in .mvn/extensions.xml writes to ~/.m2/build-cache, so + # without this it starts empty every run and never restores a module. + # Only `main` writes an entry, so pull requests warm up from `main` instead + # of each branch filling the repository-wide cache budget. + # + # The modules producing large bundled jars opt out of the cache entirely + # with false in their + # own poms, which keeps this entry around 12 MB. Without that opt-out the + # entry is 1.2 GB, 97% of it three shaded jars that account for only ~3 of + # the 27 minutes a cold build takes. + # + # Note: a cache-restored xtable-service does not reproduce + # target/quarkus-app. No step consumes that directory today; anything added + # later that does must not rely on it being present. + - name: Restore Maven build cache + id: build-cache + uses: actions/cache/restore@v6 + with: + path: ~/.m2/build-cache + key: maven-build-cache-${{ runner.os }}-jdk${{ env.JDK_VERSION }}-${{ github.sha }} + restore-keys: | + maven-build-cache-${{ runner.os }}-jdk${{ env.JDK_VERSION }}- - name: Build all module with Maven run: ./mvnw clean install -ntp -B + + # Guard against a module starting to cache large artifacts again: warn and + # skip the save rather than churn the repository-wide 10 GB cache budget, + # which is otherwise mostly the ~/.m2/repository entries above. Checking the + # size rather than specific paths keeps this working across module renames. + - name: Check Maven build cache size + id: build-cache-size + if: success() && github.ref == 'refs/heads/main' + run: | + size_mb=$(du -sm ~/.m2/build-cache 2>/dev/null | cut -f1) + size_mb=${size_mb:-0} + echo "size_mb=${size_mb}" >> "$GITHUB_OUTPUT" + echo "~/.m2/build-cache is ${size_mb} MB" + # The two bounds below are what the save step gates on, so they have to + # cover the whole range between them: 0 means the extension wrote + # nothing and there is no path for the save action to read. + if [ "${size_mb}" -eq 0 ]; then + echo "::warning title=Maven build cache not saved::~/.m2/build-cache is empty or absent. The build cache extension did not write anything; check that .mvn/extensions.xml is still in effect." + elif [ "${size_mb}" -ge 100 ]; then + echo "::warning title=Maven build cache not saved::~/.m2/build-cache is ${size_mb} MB (limit 100 MB). A module is caching large artifacts; opt it out with false in its pom." + fi + + # Skipped when the exact key already hit on restore - a re-run of the same + # commit would otherwise try to write over an immutable entry and log the + # "Unable to reserve cache with key" that splitting these jobs removed. + - name: Save Maven build cache + if: > + success() && github.ref == 'refs/heads/main' + && steps.build-cache.outputs.cache-hit != 'true' + && steps.build-cache-size.outputs.size_mb > 0 + && steps.build-cache-size.outputs.size_mb < 100 + uses: actions/cache/save@v6 + with: + path: ~/.m2/build-cache + key: maven-build-cache-${{ runner.os }}-jdk${{ env.JDK_VERSION }}-${{ github.sha }} + + # `install` above puts this project's own artifacts into ~/.m2/repository, + # including the bundled jars (~1.2 GB, xtable-utilities alone is ~1 GB). + # actions/setup-java caches that whole directory with no way to exclude a + # path, so drop them before its post step runs. They are rebuilt from + # source on every run and can never produce a useful cache hit, and a + # stale SNAPSHOT left in the local repository can shadow a reactor build. + # Source Build Check clears the same path for the same reason. + # + # This job no longer saves that cache, so nothing depends on this today. + # It is kept so the invariant holds by construction rather than by which + # step happens to write last. + - name: Drop this project's own artifacts from the Maven repository + if: always() + run: rm -rf ~/.m2/repository/org/apache/xtable diff --git a/.github/workflows/mvn-license-check.yml b/.github/workflows/mvn-license-check.yml index 889c363ba..ed6186933 100644 --- a/.github/workflows/mvn-license-check.yml +++ b/.github/workflows/mvn-license-check.yml @@ -29,18 +29,62 @@ on: branches: - "main" +# Kept in step with mvn-ci-build.yml: this job writes the cache that job reads, +# and the JDK is part of the key on both sides. +env: + JDK_VERSION: '11' + jobs: build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - name: Set up JDK 11 + - name: Set up JDK ${{ env.JDK_VERSION }} uses: actions/setup-java@v5 with: - java-version: '11' + java-version: ${{ env.JDK_VERSION }} distribution: 'temurin' - cache: maven + + # ~/.m2/repository is cached here rather than via setup-java's + # `cache: maven`, which restores on an exact key match only - it sets no + # restore-keys by design (actions/setup-java#269), so any pom change made + # every job re-download the whole dependency tree from Central. The prefix + # fallback below warms from the previous entry and downloads just the delta. + # + # This job is the writer for that cache, and deliberately the only one: + # it is the short job, so making it the writer costs the least, and it + # stops at `package`, so this project's own artifacts never enter the + # entry. Maven CI Build restores the same key read-only. + # + # Being the only writer means this job has to resolve what the readers + # need, which the license steps below do not do on their own - they build + # the shaded modules and their `-am` closure, which excludes the two + # leaves, xtable-utilities and xtable-service. The explicit resolve step + # after them covers the rest of the reactor. + # + # Restore and save are split so that only `main` writes. The entry is + # ~1 GB against a repository-wide 10 GB budget, so letting every branch + # write one would crowd out the entry they all restore from. Pull requests + # warm from `main` via the prefix and re-download only their own delta. + - name: Restore Maven repository cache + id: m2-repository + uses: actions/cache/restore@v6 + with: + path: ~/.m2/repository + key: m2-repository-${{ runner.os }}-jdk${{ env.JDK_VERSION }}-${{ hashFiles('**/pom.xml', '**/.mvn/extensions.xml') }} + restore-keys: | + m2-repository-${{ runner.os }}-jdk${{ env.JDK_VERSION }}- + + # Replaces the separate wrapper cache that setup-java's `cache: maven` + # used to provide. Keyed only on the wrapper properties, which change + # rarely, so it survives the pom changes that rotate the key above. Left + # unsplit unlike the repository cache: at ~9 MB it is not worth gating. + - name: Cache Maven wrapper + uses: actions/cache@v6 + with: + path: ~/.m2/wrapper/dists + key: m2-wrapper-${{ runner.os }}-${{ hashFiles('**/.mvn/wrapper/maven-wrapper.properties') }} - name: Apache License Check run: ./mvnw apache-rat:check -B @@ -61,3 +105,42 @@ jobs: - name: Validate Bundled License Texts run: python3 release/scripts/validate_bundled_license_texts.py + + # Nothing above this line resolves xtable-utilities or xtable-service: + # `apache-rat:check` requires no dependency resolution, `dependency:tree` + # collects poms rather than jars, and the shaded-bundle build reaches only + # the `-am` closure of the three modules it names, which excludes both + # leaves. Without this step the entry never gains the Quarkus stack or + # xtable-utilities' bundled dependencies, and Maven CI Build - which + # cannot write the cache - would re-download them from Central on every + # run, permanently. This job exists to warm the cache for that one, so it + # has to cover the whole reactor even where a license check does not. + # + # Failure here must not fail a license check, hence `|| true`: an + # incomplete cache entry costs download time on the next run and nothing + # else. `go-offline` does not catch plugin dependencies resolved at + # execution time, so expect a small delta regardless. + - name: Resolve the rest of the reactor for the cache + if: success() && github.ref == 'refs/heads/main' + run: ./mvnw -B -ntp -q dependency:go-offline -Dmaven.build.cache.enabled=false || true + + # This job writes the repository cache, so the invariant that the entry + # holds third-party dependencies only is enforced here. The steps above + # stop at `package`, so this is a no-op today; it exists so that adding + # an `install` later cannot silently add ~1.2 GB of bundled jars to the + # entry. Must stay ahead of the save below. + - name: Drop this project's own artifacts from the Maven repository + if: always() + run: rm -rf ~/.m2/repository/org/apache/xtable + + # Only when the exact key missed - entries are immutable, so saving over + # an existing key just fails. This is what actions/cache does internally + # when restore and save are not split. + - name: Save Maven repository cache + if: > + success() && github.ref == 'refs/heads/main' + && steps.m2-repository.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ~/.m2/repository + key: m2-repository-${{ runner.os }}-jdk${{ env.JDK_VERSION }}-${{ hashFiles('**/pom.xml', '**/.mvn/extensions.xml') }} diff --git a/.github/workflows/package-deploy.yml b/.github/workflows/package-deploy.yml index 7083a8f77..078d0455e 100644 --- a/.github/workflows/package-deploy.yml +++ b/.github/workflows/package-deploy.yml @@ -35,6 +35,12 @@ jobs: - name: Set Version run: ./mvnw versions:set -DnewVersion="$VERSION" - name: Publish package - run: ./mvnw -ntp --batch-mode deploy -DskipTests + # Never build a release from cached module output. `versions:set` above has + # just rewritten the version, while cache restoration is version-agnostic — + # it stamps the current project version onto whatever it restores. This + # runner keeps no build-cache directory today, so the flag is a guard + # against that changing, and it also saves checksumming every module for a + # cache that can never hit. Same reasoning as source-build-check.yml. + run: ./mvnw -ntp --batch-mode deploy -DskipTests -Dmaven.build.cache.enabled=false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 1fb0710e0..2d3d23d20 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -20,6 +20,6 @@ org.apache.maven.extensions maven-build-cache-extension - 1.1.0 + 1.2.3 diff --git a/.mvn/maven-build-cache-config.xml b/.mvn/maven-build-cache-config.xml new file mode 100644 index 000000000..172161ff7 --- /dev/null +++ b/.mvn/maven-build-cache-config.xml @@ -0,0 +1,100 @@ + + + + + true + + + + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index f7ecf7b61..da1a517c5 100644 --- a/pom.xml +++ b/pom.xml @@ -953,7 +953,8 @@ style/xml-license-header - ^<project|^<configuration|^<Configuration|^<extensions|^<component + + ^<project|^<configuration|^<Configuration|^<extensions|^<component|^<cache ^<.*xml.+?$ diff --git a/xtable-aws/pom.xml b/xtable-aws/pom.xml index 3d0a6639e..9652042df 100644 --- a/xtable-aws/pom.xml +++ b/xtable-aws/pom.xml @@ -27,6 +27,17 @@ xtable-aws XTable AWS + + + false + diff --git a/xtable-hive-metastore/pom.xml b/xtable-hive-metastore/pom.xml index 36071bfec..e6af6d15c 100644 --- a/xtable-hive-metastore/pom.xml +++ b/xtable-hive-metastore/pom.xml @@ -27,6 +27,17 @@ xtable-hive-metastore XTable HMS + + + false + diff --git a/xtable-utilities/pom.xml b/xtable-utilities/pom.xml index 53ad1497d..fd706ebfa 100644 --- a/xtable-utilities/pom.xml +++ b/xtable-utilities/pom.xml @@ -27,6 +27,17 @@ xtable-utilities_${scala.binary.version} XTable Project Utilities + + + false +