diff --git a/README.md b/README.md index 21d03d7..21a3bef 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,17 @@ with everything that `set-ci-org-and-branch`, which it sources, sets. Works under Azure Pipelines, CircleCI, GitHub Actions, and Travis CI. [Documentation](set-git-range) at top of file. +### gradlew-retry + +Runs `./gradlew` and, if Gradle could not resolve a dependency or plugin and +the failure looks transient, runs it again, by default up to 2 more times, +after 60 and then 180 seconds. This avoids CI failures when a repository such +as Maven Central transiently fails to serve an artifact. A 404 for an artifact +or plugin that does not exist, a resolution failure that does not look +transient (such as bad credentials), and other failures such as a compilation +error are not retried. +[Documentation](gradlew-retry) at top of file. + ## Search and replace ### preplace diff --git a/gradlew-retry b/gradlew-retry new file mode 100755 index 0000000..66bb144 --- /dev/null +++ b/gradlew-retry @@ -0,0 +1,117 @@ +#!/bin/bash + +# Usage: gradlew-retry [GRADLE-ARGUMENT...] +# +# Run ./gradlew (in the current directory) with the given arguments. If that fails because Gradle +# could not resolve a dependency or plugin, and the failure looks transient, wait, then run +# ./gradlew again, by default up to 2 more times. Exit with the status of the last run of +# ./gradlew. Other failures, such as a compilation error, a test failure, a build that declares +# no repositories, or a 401 status code for bad credentials, are not retried. +# +# Environment variable GRADLEW_RETRY_DELAYS, if set, is a space-separated list of the number of +# seconds to wait before each retry, such as "60 180" (the default). It also determines the +# number of retries. +# +# A resolution failure looks transient if Gradle reports a 5xx or 429 status code, a timeout, a +# refused or reset connection, or a failed TLS handshake. A repository such as Maven Central also +# sometimes transiently fails to serve an artifact that it holds, for example by returning 404 for +# a minute or more. A dependency or plugin that does not exist, such as a mistyped version or +# plugin id, also yields 404; to avoid pointless retries in that case, this script retries a 404 +# only if the artifact exists, according to the URL where Gradle looked for it, the Gradle Plugin +# Portal, Maven Central, or a Google-hosted mirror of Maven Central. This script consults the +# mirror only to make that decision; Gradle never downloads from it. This script does not retry a +# failure to find a version that matches a dynamic version such as "1.+", because the existence of +# the module's list of versions does not show that a matching version exists. +# +# Each retry passes --refresh-dependencies to Gradle, because Gradle by default caches for 24 hours +# the fact that an artifact is missing, and would otherwise not ask the repository for it again. + +# The prefix of each Maven Central URL that Gradle reports, and the corresponding prefix on the +# mirror. +central=https://repo.maven.apache.org/maven2/ +mirror=https://maven-central.storage-download.googleapis.com/maven2/ +# The prefix of each Gradle Plugin Portal URL. +plugin_portal=https://plugins.gradle.org/m2/ + +# Matches a line of Gradle output that reports a network failure that a retry might fix. +transient_regex='Received status code (5[0-9][0-9]|429)|[Tt]imed out|Connection reset|Connection refused|Remote host terminated the handshake|SSLHandshakeException|Premature end of' + +log=$(mktemp) +trap 'rm -f "$log"' EXIT + +# Return 0 (true) if the given URL serves a file. +url_exists() { + curl --silent --fail --head --location --max-time 20 --output /dev/null "$1" +} + +# Print, one per line, URLs of the pom file of the marker of each plugin that the run of Gradle +# that wrote $log reported as not found, on the Gradle Plugin Portal and on Maven Central. A +# plugin whose version Gradle did not report is omitted. +plugin_marker_urls() { + local id version path + sed -n "s/.*Plugin \[id: '\([^']*\)', version: '\([^']*\)'[^]]*\] was not found.*/\1 \2/p" "$log" \ + | while read -r id version; do + path="${id//.//}/$id.gradle.plugin/$version/$id.gradle.plugin-$version.pom" + echo "$plugin_portal$path" + echo "$central$path" + done +} + +# Return 0 (true) if the run of Gradle that wrote $log failed in a way that a retry might fix. +retry_might_help() { + if ! grep -qE 'Could not resolve all |Plugin \[id: .*\] was not found' "$log"; then + return 1 + fi + if grep -qE "$transient_regex" "$log"; then + echo "gradlew-retry: Gradle reported a network failure that a retry might fix." + return 0 + fi + # Gradle lists the URLs at which it did not find a module, one per line, as in + # " - https://repo.maven.apache.org/maven2/org/example/x/1.0/x-1.0.pom". + # A maven-metadata.xml file or a directory lists a module's versions; its existence does not + # show that a version that Gradle looked for exists, so such URLs are ignored. + local urls + urls=$({ + sed -n 's/^ *- \(https:\/\/[^ ]*\)$/\1/p' "$log" + plugin_marker_urls + } | grep -vE '/maven-metadata\.xml$|/$' | sort -u) + if [ -z "$urls" ]; then + echo "gradlew-retry: Not retrying, because the failure does not look transient." + return 1 + fi + local url + for url in $urls; do + if url_exists "$url"; then + echo "gradlew-retry: $url exists." + return 0 + fi + case "$url" in + "$central"*) + if url_exists "$mirror${url#"$central"}"; then + echo "gradlew-retry: $url exists on a mirror of Maven Central." + return 0 + fi + ;; + esac + done + echo "gradlew-retry: Not retrying, because no missing artifact exists at a URL that Gradle" + echo " reported, on the Gradle Plugin Portal, or on a mirror of Maven Central." + return 1 +} + +# Extra arguments to ./gradlew; empty for the first run. +refresh=() +for delay in ${GRADLEW_RETRY_DELAYS-60 180}; do + ./gradlew "${refresh[@]}" "$@" 2>&1 | tee "$log" + # Gradle's status, not that of `tee`. If `tee` fails, the log may be incomplete, which at worst + # prevents a retry. + status=${PIPESTATUS[0]} + if [ "$status" -eq 0 ] || ! retry_might_help; then + exit "$status" + fi + echo "gradlew-retry: Gradle could not resolve a dependency; retrying in $delay seconds." + sleep "$delay" + refresh=(--refresh-dependencies) +done +# Not `exec`, so that the EXIT trap removes $log. +./gradlew "${refresh[@]}" "$@" diff --git a/tests/Makefile b/tests/Makefile index 6f8019b..f92947f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -12,6 +12,7 @@ TEST_DIRS = \ github-organization-test \ github-pr-number-test \ github-ref-name-test \ + gradlew-retry-test \ http-status-test \ is-ci-test \ jacoco-coverage-ratchet-test \ diff --git a/tests/gradlew-retry-test/Makefile b/tests/gradlew-retry-test/Makefile new file mode 100644 index 0000000..6a8c2db --- /dev/null +++ b/tests/gradlew-retry-test/Makefile @@ -0,0 +1,8 @@ +.PHONY: all clean test + +all: test + +test: ../../gradlew-retry gradlew-retry-test.sh + ./gradlew-retry-test.sh + +clean: diff --git a/tests/gradlew-retry-test/gradlew-retry-test.sh b/tests/gradlew-retry-test/gradlew-retry-test.sh new file mode 100755 index 0000000..174b8bd --- /dev/null +++ b/tests/gradlew-retry-test/gradlew-retry-test.sh @@ -0,0 +1,181 @@ +#!/bin/sh + +# Tests `gradlew-retry`. Each case runs `gradlew-retry` in a directory whose `gradlew` is a fake +# that records each run and then succeeds or fails as the case requires. The cases that involve a +# 404 use a fake `curl` that reports only listed URLs as existing. A final smoke test queries the +# real Maven Central, and is skipped if Maven Central is unreachable. + +set -eu + +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" +GRADLEW_RETRY="$(CDPATH='' cd -- "${SCRIPT_DIR}/../.." && pwd -P)/gradlew-retry" + +# Retry without waiting. +GRADLEW_RETRY_DELAYS="0 0" +export GRADLEW_RETRY_DELAYS + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT HUP INT TERM + +status=0 + +CENTRAL=https://repo.maven.apache.org/maven2 +MIRROR=https://maven-central.storage-download.googleapis.com/maven2 +PLUGIN_PORTAL=https://plugins.gradle.org/m2 +# The path, within Maven Central, of an artifact that exists, and of one that does not. +EXISTING=org/apache/commons/commons-lang3/3.17.0/commons-lang3-3.17.0.pom +NONEXISTENT=org/apache/commons/commons-lang3/0.0.0-no-such-version/commons-lang3-0.0.0-no-such-version.pom +# The path of the marker of a plugin that exists. +EXISTING_PLUGIN_MARKER=com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/6.25.0/com.diffplug.spotless.gradle.plugin-6.25.0.pom + +# A fake `curl` that succeeds if and only if its last argument, a URL, is a line of file +# `existing-urls`. +mkdir "$work/bin" +cat > "$work/bin/curl" << EOF +#!/bin/sh +for url; do :; done +grep -qxF -- "\$url" "$work/existing-urls" +EOF +chmod +x "$work/bin/curl" +# The PATH for running gradlew-retry. Initially, it finds the fake `curl`. +check_path="$work/bin:$PATH" + +# existing_urls URL...: makes the fake `curl` report exactly the given URLs as existing. +existing_urls() { + : > "$work/existing-urls" + for url in "$@"; do + echo "$url" >> "$work/existing-urls" + done +} +existing_urls + +# make_gradlew BODY: makes ./gradlew a script that records the run, as a line containing its +# arguments, in file `runs` and then runs BODY, a shell command. Within BODY, $run is the number of +# this run, starting at 1. +make_gradlew() { + cat > "$work/gradlew" << EOF +#!/bin/sh +echo "\$*" >> runs +run=\$(wc -l < runs) +$1 +EOF + chmod +x "$work/gradlew" +} + +# not_found_output PATH: prints Gradle's output when it finds a module at neither Maven Central +# nor the Gradle Plugin Portal, given the path of the module's pom file within each repository. +not_found_output() { + cat << EOF +FAILURE: Build failed with an exception. +> Could not resolve all artifacts for configuration 'classpath'. + > Could not find org.example:example:1.0. + Searched in the following locations: + - $CENTRAL/$1 + - $PLUGIN_PORTAL/$1 +EOF +} + +# check DESCRIPTION EXPECTED-STATUS EXPECTED-RUNS: runs `gradlew-retry build` in the work directory +# and reports whether its exit status and the number of runs of ./gradlew are as expected. +check() { + rm -f "$work/runs" + actual_status=0 + (cd "$work" && PATH="$check_path" "$GRADLEW_RETRY" build > output 2>&1) || actual_status=$? + actual_runs=$(wc -l < "$work/runs" | tr -d ' ') + if [ "$actual_status" = "$2" ] && [ "$actual_runs" = "$3" ]; then + echo "PASS: $1" + else + echo "FAIL: $1" + echo " expected: status $2, $3 runs" + echo " actual: status $actual_status, $actual_runs runs" + sed 's/^/ | /' "$work/output" + status=1 + fi +} + +make_gradlew 'exit 0' +check "success is not retried" 0 1 + +make_gradlew 'echo "FAILURE: Build failed with an exception."; exit 7' +check "a failure other than resolution is not retried, and its status is kept" 7 1 + +make_gradlew 'echo "> Could not resolve all files for configuration x." +echo " > Could not GET '\''https://example.org/x.pom'\''. Received status code 502"; exit 1' +check "a 5xx status is retried twice" 1 3 + +make_gradlew 'echo "> Could not resolve all files for configuration x." +echo " > Could not GET '\''https://example.org/x.pom'\''." +echo " > Read timed out"; exit 1' +check "a timeout is retried twice" 1 3 + +make_gradlew 'echo "> Could not resolve all files for configuration x." +echo " > Cannot resolve external dependency org.example:x:1.0 because no repositories are defined."; exit 1' +check "a build that declares no repositories is not retried" 1 1 + +make_gradlew 'echo "> Could not resolve all files for configuration x." +echo " > Could not GET '\''https://example.org/x.pom'\''. Received status code 401 from server: Unauthorized"; exit 1' +check "a 401 status is not retried" 1 1 + +make_gradlew 'echo "Plugin [id: '\''org.example'\''] was not found in any of the following sources:"; exit 1' +check "a missing plugin without a version is not retried" 1 1 + +not_found_output "$EXISTING" > "$work/existing.txt" +not_found_output "$NONEXISTENT" > "$work/nonexistent.txt" + +existing_urls "$CENTRAL/$EXISTING" +# shellcheck disable=SC2016 # The fake ./gradlew, not this script, expands $run. +make_gradlew '[ "$run" -ge 2 ] && exit 0; cat existing.txt; exit 1' +check "a transient 404 is retried until it succeeds" 0 2 +if [ "$(cat "$work/runs")" = "$(printf 'build\n--refresh-dependencies build')" ]; then + echo "PASS: a retry passes --refresh-dependencies" +else + echo "FAIL: a retry passes --refresh-dependencies" + sed 's/^/ | /' "$work/runs" + status=1 +fi + +make_gradlew 'cat existing.txt; exit 1' +check "a persistent 404 for an existing artifact is retried twice" 1 3 + +existing_urls "$MIRROR/$EXISTING" +check "a 404 for an artifact that exists only on the mirror is retried twice" 1 3 + +existing_urls +check "a 404 for an artifact on neither Maven Central nor the mirror is not retried" 1 1 + +cat > "$work/dynamic.txt" << EOF +> Could not resolve all files for configuration ':compileClasspath'. + > Could not find any version that matches org.apache.commons:commons-lang3:99.+. + Versions that do not match: + - 3.17.0 + Searched in the following locations: + - $CENTRAL/org/apache/commons/commons-lang3/maven-metadata.xml + - $CENTRAL/org/apache/commons/commons-lang3/ +EOF +existing_urls "$CENTRAL/org/apache/commons/commons-lang3/maven-metadata.xml" \ + "$CENTRAL/org/apache/commons/commons-lang3/" +make_gradlew 'cat dynamic.txt; exit 1' +check "an unmatched dynamic version is not retried" 1 1 + +make_gradlew 'echo "Plugin [id: '\''com.diffplug.spotless'\'', version: '\''6.25.0'\'', apply: false] was not found in any of the following sources:"; exit 1' +existing_urls "$PLUGIN_PORTAL/$EXISTING_PLUGIN_MARKER" +check "a missing plugin that exists on the Plugin Portal is retried twice" 1 3 +existing_urls "$MIRROR/$EXISTING_PLUGIN_MARKER" +check "a missing plugin whose marker exists only on the mirror is retried twice" 1 3 +existing_urls +check "a missing plugin that does not exist is not retried" 1 1 + +# Smoke test against the real repositories, using the real `curl`. +check_path="$PATH" +if ! curl --silent --fail --head --max-time 20 --output /dev/null "$CENTRAL/$EXISTING"; then + echo "SKIP: gradlew-retry smoke test: Maven Central is unreachable" + exit "$status" +fi + +make_gradlew 'cat existing.txt; exit 1' +check "smoke test: a 404 for an existing artifact is retried twice" 1 3 + +make_gradlew 'cat nonexistent.txt; exit 1' +check "smoke test: a 404 for a nonexistent artifact is not retried" 1 1 + +exit "$status"