Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,17 @@ with everything that `set-ci-org-and-branch`, which it sources, sets.
Works under Azure Pipelines, CircleCI, GitHub Actions, and Travis CI.
[Documentation](set-git-range) at top of file.

### gradlew-retry

Runs `./gradlew` and, if Gradle could not resolve a dependency or plugin and
the failure looks transient, runs it again, by default up to 2 more times,
after 60 and then 180 seconds. This avoids CI failures when a repository such
as Maven Central transiently fails to serve an artifact. A 404 for an artifact
or plugin that does not exist, a resolution failure that does not look
transient (such as bad credentials), and other failures such as a compilation
error are not retried.
[Documentation](gradlew-retry) at top of file.

## Search and replace

### preplace
Expand Down
117 changes: 117 additions & 0 deletions gradlew-retry
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
#!/bin/bash

# Usage: gradlew-retry [GRADLE-ARGUMENT...]
#
# Run ./gradlew (in the current directory) with the given arguments. If that fails because Gradle
# could not resolve a dependency or plugin, and the failure looks transient, wait, then run
# ./gradlew again, by default up to 2 more times. Exit with the status of the last run of
# ./gradlew. Other failures, such as a compilation error, a test failure, a build that declares
# no repositories, or a 401 status code for bad credentials, are not retried.
#
# Environment variable GRADLEW_RETRY_DELAYS, if set, is a space-separated list of the number of
# seconds to wait before each retry, such as "60 180" (the default). It also determines the
# number of retries.
#
# A resolution failure looks transient if Gradle reports a 5xx or 429 status code, a timeout, a
# refused or reset connection, or a failed TLS handshake. A repository such as Maven Central also
# sometimes transiently fails to serve an artifact that it holds, for example by returning 404 for
# a minute or more. A dependency or plugin that does not exist, such as a mistyped version or
# plugin id, also yields 404; to avoid pointless retries in that case, this script retries a 404
# only if the artifact exists, according to the URL where Gradle looked for it, the Gradle Plugin
# Portal, Maven Central, or a Google-hosted mirror of Maven Central. This script consults the
# mirror only to make that decision; Gradle never downloads from it. This script does not retry a
# failure to find a version that matches a dynamic version such as "1.+", because the existence of
# the module's list of versions does not show that a matching version exists.
#
# Each retry passes --refresh-dependencies to Gradle, because Gradle by default caches for 24 hours
# the fact that an artifact is missing, and would otherwise not ask the repository for it again.

# The prefix of each Maven Central URL that Gradle reports, and the corresponding prefix on the
# mirror.
central=https://repo.maven.apache.org/maven2/
mirror=https://maven-central.storage-download.googleapis.com/maven2/
# The prefix of each Gradle Plugin Portal URL.
plugin_portal=https://plugins.gradle.org/m2/

# Matches a line of Gradle output that reports a network failure that a retry might fix.
transient_regex='Received status code (5[0-9][0-9]|429)|[Tt]imed out|Connection reset|Connection refused|Remote host terminated the handshake|SSLHandshakeException|Premature end of'

log=$(mktemp)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
trap 'rm -f "$log"' EXIT

# Return 0 (true) if the given URL serves a file.
url_exists() {
curl --silent --fail --head --location --max-time 20 --output /dev/null "$1"
}

# Print, one per line, URLs of the pom file of the marker of each plugin that the run of Gradle
# that wrote $log reported as not found, on the Gradle Plugin Portal and on Maven Central. A
# plugin whose version Gradle did not report is omitted.
plugin_marker_urls() {
local id version path
sed -n "s/.*Plugin \[id: '\([^']*\)', version: '\([^']*\)'[^]]*\] was not found.*/\1 \2/p" "$log" \
| while read -r id version; do
path="${id//.//}/$id.gradle.plugin/$version/$id.gradle.plugin-$version.pom"
echo "$plugin_portal$path"
echo "$central$path"
done
}

# Return 0 (true) if the run of Gradle that wrote $log failed in a way that a retry might fix.
retry_might_help() {
if ! grep -qE 'Could not resolve all |Plugin \[id: .*\] was not found' "$log"; then
return 1
fi
if grep -qE "$transient_regex" "$log"; then
echo "gradlew-retry: Gradle reported a network failure that a retry might fix."
return 0
fi
# Gradle lists the URLs at which it did not find a module, one per line, as in
# " - https://repo.maven.apache.org/maven2/org/example/x/1.0/x-1.0.pom".
# A maven-metadata.xml file or a directory lists a module's versions; its existence does not
# show that a version that Gradle looked for exists, so such URLs are ignored.
local urls
urls=$({
sed -n 's/^ *- \(https:\/\/[^ ]*\)$/\1/p' "$log"
plugin_marker_urls
} | grep -vE '/maven-metadata\.xml$|/$' | sort -u)
if [ -z "$urls" ]; then
echo "gradlew-retry: Not retrying, because the failure does not look transient."
return 1
fi
local url
for url in $urls; do
if url_exists "$url"; then
echo "gradlew-retry: $url exists."
return 0
fi
case "$url" in
"$central"*)
if url_exists "$mirror${url#"$central"}"; then
echo "gradlew-retry: $url exists on a mirror of Maven Central."
return 0
fi
;;
esac
done
echo "gradlew-retry: Not retrying, because no missing artifact exists at a URL that Gradle"
echo " reported, on the Gradle Plugin Portal, or on a mirror of Maven Central."
return 1
}

# Extra arguments to ./gradlew; empty for the first run.
refresh=()
for delay in ${GRADLEW_RETRY_DELAYS-60 180}; do
./gradlew "${refresh[@]}" "$@" 2>&1 | tee "$log"
# Gradle's status, not that of `tee`. If `tee` fails, the log may be incomplete, which at worst
# prevents a retry.
status=${PIPESTATUS[0]}
if [ "$status" -eq 0 ] || ! retry_might_help; then
exit "$status"
fi
echo "gradlew-retry: Gradle could not resolve a dependency; retrying in $delay seconds."
sleep "$delay"
refresh=(--refresh-dependencies)
done
# Not `exec`, so that the EXIT trap removes $log.
./gradlew "${refresh[@]}" "$@"
1 change: 1 addition & 0 deletions tests/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ TEST_DIRS = \
github-organization-test \
github-pr-number-test \
github-ref-name-test \
gradlew-retry-test \
http-status-test \
is-ci-test \
jacoco-coverage-ratchet-test \
Expand Down
8 changes: 8 additions & 0 deletions tests/gradlew-retry-test/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
.PHONY: all clean test

all: test

test: ../../gradlew-retry gradlew-retry-test.sh
./gradlew-retry-test.sh

clean:
181 changes: 181 additions & 0 deletions tests/gradlew-retry-test/gradlew-retry-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
#!/bin/sh

# Tests `gradlew-retry`. Each case runs `gradlew-retry` in a directory whose `gradlew` is a fake
# that records each run and then succeeds or fails as the case requires. The cases that involve a
# 404 use a fake `curl` that reports only listed URLs as existing. A final smoke test queries the
# real Maven Central, and is skipped if Maven Central is unreachable.

set -eu

SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)"
GRADLEW_RETRY="$(CDPATH='' cd -- "${SCRIPT_DIR}/../.." && pwd -P)/gradlew-retry"

# Retry without waiting.
GRADLEW_RETRY_DELAYS="0 0"
export GRADLEW_RETRY_DELAYS

work="$(mktemp -d)"
trap 'rm -rf "$work"' EXIT HUP INT TERM

status=0

CENTRAL=https://repo.maven.apache.org/maven2
MIRROR=https://maven-central.storage-download.googleapis.com/maven2
PLUGIN_PORTAL=https://plugins.gradle.org/m2
# The path, within Maven Central, of an artifact that exists, and of one that does not.
EXISTING=org/apache/commons/commons-lang3/3.17.0/commons-lang3-3.17.0.pom
NONEXISTENT=org/apache/commons/commons-lang3/0.0.0-no-such-version/commons-lang3-0.0.0-no-such-version.pom
# The path of the marker of a plugin that exists.
EXISTING_PLUGIN_MARKER=com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/6.25.0/com.diffplug.spotless.gradle.plugin-6.25.0.pom

# A fake `curl` that succeeds if and only if its last argument, a URL, is a line of file
# `existing-urls`.
mkdir "$work/bin"
cat > "$work/bin/curl" << EOF
#!/bin/sh
for url; do :; done
grep -qxF -- "\$url" "$work/existing-urls"
EOF
chmod +x "$work/bin/curl"
# The PATH for running gradlew-retry. Initially, it finds the fake `curl`.
check_path="$work/bin:$PATH"

# existing_urls URL...: makes the fake `curl` report exactly the given URLs as existing.
existing_urls() {
: > "$work/existing-urls"
for url in "$@"; do
echo "$url" >> "$work/existing-urls"
done
}
existing_urls

# make_gradlew BODY: makes ./gradlew a script that records the run, as a line containing its
# arguments, in file `runs` and then runs BODY, a shell command. Within BODY, $run is the number of
# this run, starting at 1.
make_gradlew() {
cat > "$work/gradlew" << EOF
#!/bin/sh
echo "\$*" >> runs
run=\$(wc -l < runs)
$1
EOF
chmod +x "$work/gradlew"
}

# not_found_output PATH: prints Gradle's output when it finds a module at neither Maven Central
# nor the Gradle Plugin Portal, given the path of the module's pom file within each repository.
not_found_output() {
cat << EOF
FAILURE: Build failed with an exception.
> Could not resolve all artifacts for configuration 'classpath'.
> Could not find org.example:example:1.0.
Searched in the following locations:
- $CENTRAL/$1
- $PLUGIN_PORTAL/$1
EOF
}

# check DESCRIPTION EXPECTED-STATUS EXPECTED-RUNS: runs `gradlew-retry build` in the work directory
# and reports whether its exit status and the number of runs of ./gradlew are as expected.
check() {
rm -f "$work/runs"
actual_status=0
(cd "$work" && PATH="$check_path" "$GRADLEW_RETRY" build > output 2>&1) || actual_status=$?
actual_runs=$(wc -l < "$work/runs" | tr -d ' ')
if [ "$actual_status" = "$2" ] && [ "$actual_runs" = "$3" ]; then
echo "PASS: $1"
else
echo "FAIL: $1"
echo " expected: status $2, $3 runs"
echo " actual: status $actual_status, $actual_runs runs"
sed 's/^/ | /' "$work/output"
status=1
fi
}

make_gradlew 'exit 0'
check "success is not retried" 0 1

make_gradlew 'echo "FAILURE: Build failed with an exception."; exit 7'
check "a failure other than resolution is not retried, and its status is kept" 7 1

make_gradlew 'echo "> Could not resolve all files for configuration x."
echo " > Could not GET '\''https://example.org/x.pom'\''. Received status code 502"; exit 1'
check "a 5xx status is retried twice" 1 3

make_gradlew 'echo "> Could not resolve all files for configuration x."
echo " > Could not GET '\''https://example.org/x.pom'\''."
echo " > Read timed out"; exit 1'
check "a timeout is retried twice" 1 3

make_gradlew 'echo "> Could not resolve all files for configuration x."
echo " > Cannot resolve external dependency org.example:x:1.0 because no repositories are defined."; exit 1'
check "a build that declares no repositories is not retried" 1 1

make_gradlew 'echo "> Could not resolve all files for configuration x."
echo " > Could not GET '\''https://example.org/x.pom'\''. Received status code 401 from server: Unauthorized"; exit 1'
check "a 401 status is not retried" 1 1

make_gradlew 'echo "Plugin [id: '\''org.example'\''] was not found in any of the following sources:"; exit 1'
check "a missing plugin without a version is not retried" 1 1

not_found_output "$EXISTING" > "$work/existing.txt"
not_found_output "$NONEXISTENT" > "$work/nonexistent.txt"

existing_urls "$CENTRAL/$EXISTING"
# shellcheck disable=SC2016 # The fake ./gradlew, not this script, expands $run.
make_gradlew '[ "$run" -ge 2 ] && exit 0; cat existing.txt; exit 1'
check "a transient 404 is retried until it succeeds" 0 2
if [ "$(cat "$work/runs")" = "$(printf 'build\n--refresh-dependencies build')" ]; then
echo "PASS: a retry passes --refresh-dependencies"
else
echo "FAIL: a retry passes --refresh-dependencies"
sed 's/^/ | /' "$work/runs"
status=1
fi

make_gradlew 'cat existing.txt; exit 1'
check "a persistent 404 for an existing artifact is retried twice" 1 3

existing_urls "$MIRROR/$EXISTING"
check "a 404 for an artifact that exists only on the mirror is retried twice" 1 3

existing_urls
check "a 404 for an artifact on neither Maven Central nor the mirror is not retried" 1 1

cat > "$work/dynamic.txt" << EOF
> Could not resolve all files for configuration ':compileClasspath'.
> Could not find any version that matches org.apache.commons:commons-lang3:99.+.
Versions that do not match:
- 3.17.0
Searched in the following locations:
- $CENTRAL/org/apache/commons/commons-lang3/maven-metadata.xml
- $CENTRAL/org/apache/commons/commons-lang3/
EOF
existing_urls "$CENTRAL/org/apache/commons/commons-lang3/maven-metadata.xml" \
"$CENTRAL/org/apache/commons/commons-lang3/"
make_gradlew 'cat dynamic.txt; exit 1'
check "an unmatched dynamic version is not retried" 1 1

make_gradlew 'echo "Plugin [id: '\''com.diffplug.spotless'\'', version: '\''6.25.0'\'', apply: false] was not found in any of the following sources:"; exit 1'
existing_urls "$PLUGIN_PORTAL/$EXISTING_PLUGIN_MARKER"
check "a missing plugin that exists on the Plugin Portal is retried twice" 1 3
existing_urls "$MIRROR/$EXISTING_PLUGIN_MARKER"
check "a missing plugin whose marker exists only on the mirror is retried twice" 1 3
existing_urls
check "a missing plugin that does not exist is not retried" 1 1

# Smoke test against the real repositories, using the real `curl`.
check_path="$PATH"
if ! curl --silent --fail --head --max-time 20 --output /dev/null "$CENTRAL/$EXISTING"; then
echo "SKIP: gradlew-retry smoke test: Maven Central is unreachable"
exit "$status"
fi

make_gradlew 'cat existing.txt; exit 1'
check "smoke test: a 404 for an existing artifact is retried twice" 1 3

make_gradlew 'cat nonexistent.txt; exit 1'
check "smoke test: a 404 for a nonexistent artifact is not retried" 1 1

exit "$status"
Loading