diff --git a/docs/How-to/optionalmodules_installation.md b/docs/How-to/optionalmodules_installation.md index 6b4fdd821..6600b67c4 100644 --- a/docs/How-to/optionalmodules_installation.md +++ b/docs/How-to/optionalmodules_installation.md @@ -21,7 +21,8 @@ bash tools/get_spider.sh === "Linux" - The PETSc script downloads a pre-compiled version from OSF and configures + The PETSc script downloads the PETSc 3.19.0 source archive from Zenodo (or its + DataverseNL mirror), checks its SHA-256, builds it and configures `PETSC_DIR` and `PETSC_ARCH` automatically. !!! note "Fedora / RHEL" diff --git a/docs/How-to/update_module_pins.md b/docs/How-to/update_module_pins.md index e89392389..e819287ba 100644 --- a/docs/How-to/update_module_pins.md +++ b/docs/How-to/update_module_pins.md @@ -23,8 +23,9 @@ module is distributed. A third entry, PETSc, is pinned in `[tool.proteus.modules.petsc]` by the SHA-256 -of a pre-built archive rather than a git ref, because it is downloaded as a -binary from OSF. +of its source archive rather than a git ref: `tools/get_petsc.sh` downloads it from +Zenodo (`url`), or from the DataverseNL mirror (`mirror`) when Zenodo fails, and +refuses a file with another SHA-256. **PyPI floors** are minimum bounds. When a user runs `pip install -e ".[develop]"`, pip resolves the newest release that satisfies every floor. The floor is the @@ -37,8 +38,8 @@ identical AGNI, SOCRATES, and SPIDER checkouts. ## How the update process works -Everything downstream reads from the same table, so a pin only needs to be -changed in one place: +Everything in PROTEUS reads from the same table, so a pin is changed in +`pyproject.toml` alone (PETSc is the exception, see Case 4): - **`tools/get_*.sh`** read git pins through `tools/_module_pins.py` (`python tools/_module_pins.py agni ref` prints the pinned AGNI commit) and @@ -55,7 +56,8 @@ changed in one place: pinned but not drift-checked by the doctor. Because of this, bumping a module is a single-line edit followed by a badge -refresh and a local re-sync. +refresh and a local re-sync; a PETSc archive change is three lines here and the +matching lines in SPIDER. ## Procedure @@ -177,6 +179,25 @@ follow upstream, not a default. cannot diverge. Do not add a second pin for these in `[tool.proteus.modules]`. +### Case 4: change the PETSc archive + +PETSc is pinned by its archive, not a ref: `url` (the Zenodo file), `mirror` +(the DataverseNL file of the same archive) and `sha256` in +`[tool.proteus.modules.petsc]` change together, since `tools/get_petsc.sh` +accepts a file from either source only when its SHA-256 matches. SPIDER's +`tools/get_petsc.sh` holds its own copy of the three values and is changed in the +same window, so the two installs build the same PETSc. + +Check the mirror on its own by breaking the Zenodo source, so the download has +to come from DataverseNL: + +```console +PETSC_URL=file:///nonexistent/petsc.zip bash tools/get_petsc.sh /tmp/petsc-check +``` + +The log names the DataverseNL file it downloaded, and the build continues only if +its SHA-256 matches the pin. + ## Propagating the change to other developers Once the pin change is merged, other developers pick it up the same way a user diff --git a/pyproject.toml b/pyproject.toml index 7c8086699..4380954b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -323,10 +323,11 @@ url = "https://github.com/FormingWorlds/LavAtmos" ref = "c963dd2acf6c8a4c33d246d653342121666ae612" [tool.proteus.modules.petsc] -# Pre-built PETSc archive on OSF. Required by SPIDER. tools/get_petsc.sh -# downloads and validates by SHA-256. -url = "https://osf.io/download/p5vxq/" -sha256 = "" +# PETSc 3.19.0 source archive (Zenodo 15805756), required by SPIDER. tools/get_petsc.sh +# tries url, then the DataverseNL mirror (10.34894/BZDJPU), and checks the SHA-256. +url = "https://zenodo.org/records/15805756/files/petsc.zip?download=1" +mirror = "https://dataverse.nl/api/access/datafile/683669" +sha256 = "c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d" [tool.proteus.coverage_fast] # Fast PR gate: unit-only coverage on PRs (smoke, integration, and slow tiers diff --git a/tests/tools/test_install_scripts.py b/tests/tools/test_install_scripts.py index e2de6374b..c14088098 100644 --- a/tests/tools/test_install_scripts.py +++ b/tests/tools/test_install_scripts.py @@ -635,6 +635,165 @@ def test_resolve_module_pin_reads_the_pin_and_stops_when_it_is_missing(tmp_path) assert 'REACHED_CLONE' not in missing.stdout +def _sha256(data: bytes) -> str: + """Return the SHA-256 hex digest of ``data``.""" + import hashlib + + return hashlib.sha256(data).hexdigest() + + +@pytest.mark.unit +@pytest.mark.parametrize('strict', [False, True]) +def test_fetch_verified_falls_back_to_the_next_source_with_the_pinned_hash(tmp_path, strict): + """A failed download, an empty url and a wrong file each give way to the next source. + + The archive goes straight into ``unzip`` and a PETSc build, so only a file + with the pinned SHA-256 may end the search. + """ + good, bad = tmp_path / 'good.zip', tmp_path / 'bad.zip' + good.write_bytes(b'petsc archive') + bad.write_bytes(b'an error page') + dest = tmp_path / 'petsc.zip' + body = ( + f'fetch_verified {_sha256(b"petsc archive")} "{dest}" ' + f'"file://{tmp_path}/missing.zip" "" "file://{bad}" "file://{good}"\n' + 'echo REACHED_UNZIP\n' + ) + + result = _run_bash(_with_common(body, strict)) + + assert result.returncode == 0, result.stderr + assert dest.read_bytes() == b'petsc archive' + assert 'REACHED_UNZIP' in result.stdout + assert f'download from file://{tmp_path}/missing.zip failed' in result.stderr + assert ( + f'file://{bad} served a file with SHA-256 {_sha256(b"an error page")}' in result.stderr + ) + + +@pytest.mark.unit +@pytest.mark.parametrize('strict', [False, True]) +def test_fetch_verified_stops_when_no_source_serves_the_pinned_file(tmp_path, strict): + """With no matching source it returns 1 and leaves no archive, so a get_* script's + ERR trap prints its troubleshooting text and the install stops before unzip.""" + bad = tmp_path / 'bad.zip' + bad.write_bytes(b'an error page') + dest = tmp_path / 'petsc.zip' + call = f'fetch_verified {"0" * 64} "{dest}" "file://{bad}" ""' + + returned = _run_bash(_with_common(f'{call} || echo "RETURNED $?"\n', strict)) + trapped = _run_bash( + _with_common(f'set -e\ntrap "echo TRAPPED" ERR\n{call}\necho REACHED_UNZIP\n', strict) + ) + + assert 'RETURNED 1' in returned.stdout + assert not dest.exists() + assert f'no source served petsc.zip with SHA-256 {"0" * 64}' in returned.stderr + assert 'download from failed' not in returned.stderr + assert trapped.returncode == 1 + assert 'TRAPPED' in trapped.stdout and 'REACHED_UNZIP' not in trapped.stdout + + +@pytest.mark.unit +def test_fetch_verified_refuses_an_empty_hash_or_no_hash_tool(tmp_path): + """A pin that could not be read, or no shasum and no sha256sum, stops before any download.""" + good = tmp_path / 'good.zip' + good.write_bytes(b'petsc archive') + call = 'fetch_verified "{}" "' + f'{tmp_path}/p.zip" "file://{good}"\n' + blank = _run_bash(_with_common(call.format(''))) + no_tool = subprocess.run( + ['/bin/bash', '-c', _with_common(call.format(_sha256(b'petsc archive')))], + capture_output=True, + text=True, + env={'PATH': str(tmp_path / 'empty')}, + ) + assert blank.returncode == 1 and 'no SHA-256 pin for p.zip' in blank.stderr + assert no_tool.returncode == 1 + assert 'neither shasum nor sha256sum is installed' in no_tool.stderr + for result in (blank, no_tool): + assert 'Downloading' not in result.stdout + + +@pytest.mark.unit +def test_get_petsc_tries_zenodo_then_the_mirror_from_its_pins(tmp_path): + """The shipped download step passes the pinned hash and sources, overridable by + PETSC_URL and PETSC_MIRROR_URL, to fetch_verified in that order.""" + fake_tools = tmp_path / 'tools' + fake_tools.mkdir() + shutil.copy2(COMMON_LIB, fake_tools / '_get_common.sh') + (fake_tools / '_module_pins.py').write_text('') + archive = tmp_path / 'mirror.zip' + archive.write_bytes(b'petsc archive') + wrong = tmp_path / 'wrong.zip' + wrong.write_bytes(b'an error page') + stubs = tmp_path / 'stubs' + stubs.mkdir() + _write_stub( + stubs, + 'python', + f'#!/bin/bash\ncase "$3" in sha256) echo {_sha256(b"petsc archive")} ;; ' + f'url) echo "file://{wrong}" ;; mirror) echo "$PIN_MIRROR" ;; esac\n', + ) + block = _extract_script_block( + 'get_petsc.sh', '# Read the archive pins', 'current_step="Decompressing' + ) + work = tmp_path / 'work' + snippet = ( + f'set -e\nsource "{fake_tools}/_get_common.sh"\nworkpath="{work}"\n{block}\necho DONE\n' + ) + env = {**os.environ, 'PATH': f'{stubs}:{os.environ["PATH"]}', 'PIN_MIRROR': ''} + + def run(**extra): + return subprocess.run( + ['bash', '-c', snippet], capture_output=True, text=True, env={**env, **extra} + ) + + # Both pinned sources serve a wrong file: the install stops. + pinned = run(PIN_MIRROR=f'file://{wrong}') + assert pinned.returncode == 1 + assert pinned.stderr.count(f'file://{wrong} served a file') == 2 + + # An empty pin stops with the cause instead of skipping that source. + empty_mirror = run() + assert empty_mirror.returncode == 1 + assert 'cannot read the PETSc url, mirror and sha256' in empty_mirror.stderr + + # Zenodo first, then the mirror. + mirrored = run(PIN_MIRROR=f'file://{archive}') + assert mirrored.returncode == 0, mirrored.stderr + out = mirrored.stdout + assert out.index(f'Downloading file://{wrong}') < out.index(f'Downloading file://{archive}') + assert (work / 'petsc.zip').read_bytes() == b'petsc archive' + assert 'DONE' in out + + # PETSC_URL replaces the pinned url, and PETSC_MIRROR_URL the pinned mirror. + overridden = run(PETSC_URL=f'file://{archive}', PIN_MIRROR=f'file://{wrong}') + assert overridden.returncode == 0, overridden.stderr + assert f'Downloading file://{wrong}' not in overridden.stdout + mirror_set = run(PETSC_MIRROR_URL=f'file://{archive}', PIN_MIRROR=f'file://{wrong}') + assert mirror_set.returncode == 0, mirror_set.stderr + + # A pin that cannot be read stops before the previous installation is removed. + (work / 'old_build').write_text('kept') + _write_stub(stubs, 'python', '#!/bin/bash\nexit 1\n') + unreadable = run() + assert unreadable.returncode == 1 + assert 'needs python 3.11 or newer (tomllib)' in unreadable.stderr + assert (work / 'old_build').read_text() == 'kept' + assert 'Downloading' not in unreadable.stdout + + +@pytest.mark.unit +def test_pyproject_pins_the_petsc_archive_by_sha256(): + """PETSc is pinned to its Zenodo archive by a SHA-256, with the DataverseNL file as mirror.""" + pin = tomllib.loads((TOOLS_DIR.parent / 'pyproject.toml').read_text())['tool']['proteus'][ + 'modules' + ]['petsc'] + assert pin['url'] == 'https://zenodo.org/records/15805756/files/petsc.zip?download=1' + assert pin['sha256'] == 'c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d' + assert pin['mirror'] == 'https://dataverse.nl/api/access/datafile/683669' + + # --------------------------------------------------------------------------- # ERR trap tests # --------------------------------------------------------------------------- @@ -674,7 +833,7 @@ def test_err_trap_reports_current_step(): """ERR trap output includes the ``current_step`` variable value.""" snippet = """\ set -e -current_step="Downloading PETSc archive from OSF" +current_step="Downloading PETSc archive" on_error() { local rc=$? echo "STEP=$current_step" @@ -688,7 +847,7 @@ def test_err_trap_reports_current_step(): text=True, ) assert result.returncode != 0 - assert 'STEP=Downloading PETSc archive from OSF' in result.stdout + assert 'STEP=Downloading PETSc archive' in result.stdout # --------------------------------------------------------------------------- diff --git a/tools/_get_common.sh b/tools/_get_common.sh index 1425485ce..2058d488f 100644 --- a/tools/_get_common.sh +++ b/tools/_get_common.sh @@ -170,3 +170,34 @@ resolve_module_pin() { exit 1 fi } + +# Download from the first of the urls that serves a file with the given +# SHA-256, skipping an empty url. Exits 1, with every url's failure named on +# stderr, when none does. +fetch_verified() { + local sha256="$1" dest="$2" url got + shift 2 + if [ -z "$sha256" ]; then + echo "ERROR: no SHA-256 pin for $(basename "$dest"); check tools/_module_pins.py" >&2 + return 1 + fi + if ! command -v shasum >/dev/null 2>&1 && ! command -v sha256sum >/dev/null 2>&1; then + echo "ERROR: neither shasum nor sha256sum is installed; install one of them" >&2 + return 1 + fi + for url in "$@"; do + [ -n "$url" ] || continue + echo "Downloading $url" + if curl -fLsS --retry 3 --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ + "$url" -o "$dest"; then + got=$( (shasum -a 256 "$dest" 2>/dev/null || sha256sum "$dest") | awk '{print $1}') + [ "$got" = "$sha256" ] && return 0 + echo "WARNING: $url served a file with SHA-256 $got, not $sha256" >&2 + else + echo "WARNING: download from $url failed" >&2 + fi + done + rm -f "$dest" + echo "ERROR: no source served $(basename "$dest") with SHA-256 $sha256" >&2 + return 1 +} diff --git a/tools/_module_pins.py b/tools/_module_pins.py index 8ff79cbb1..6dd84f4f1 100644 --- a/tools/_module_pins.py +++ b/tools/_module_pins.py @@ -2,13 +2,14 @@ Single entry point for tools/get_*.sh and the CI composite action to look up the URL and ref of an external module (AGNI, SOCRATES, SPIDER, VULCAN, -LovePy, PETSc). The source of truth is pyproject.toml's +LovePy), or the url, mirror and sha256 of the PETSc archive. The source of truth is pyproject.toml's ``[tool.proteus.modules.]`` table. Usage from a shell script: URL=$(python tools/_module_pins.py agni url) REF=$(python tools/_module_pins.py agni ref) + MIRROR=$(python tools/_module_pins.py petsc mirror) Exits non-zero with a clear message if the module is unknown or the requested field is missing. diff --git a/tools/get_petsc.sh b/tools/get_petsc.sh index ab81b3837..dc6cfc229 100755 --- a/tools/get_petsc.sh +++ b/tools/get_petsc.sh @@ -3,7 +3,8 @@ # get_petsc.sh — Download, configure, and compile PETSc for PROTEUS/SPIDER # ============================================================================= # -# Downloads PETSc 3.19.0 from OSF and builds it with sundials2 support. +# Downloads the PETSc 3.19.0 source archive from Zenodo, or its DataverseNL +# mirror, checks its SHA-256, and builds it with sundials2 support. # SPIDER is a pure C code, so C++ and Fortran compilers are disabled. # # Supported platforms: @@ -52,8 +53,8 @@ on_error() { case "$current_step" in *"Download"*) echo " - Check your internet connection" - echo " - Verify the OSF URL is accessible: $url" - echo " - Try downloading manually: curl -LsS $url > petsc.zip" + echo " - The sources and the SHA-256 are in pyproject.toml [tool.proteus.modules.petsc];" + echo " PETSC_URL and PETSC_MIRROR_URL override the sources" ;; *"Decompress"*) echo " - The downloaded archive may be corrupted" @@ -117,21 +118,29 @@ export PETSC_DIR="$workpath" echo "PETSC_DIR = $PETSC_DIR" echo "PETSC_ARCH = $PETSC_ARCH" +# Read the archive pins before the previous installation is removed +petsc_pin() { python "$proteus_tools_dir/_module_pins.py" petsc "$1"; } +petsc_sha256=$(petsc_pin sha256) || petsc_sha256="" +petsc_url=$(petsc_pin url) || petsc_url="" +petsc_mirror=$(petsc_pin mirror) || petsc_mirror="" +if [[ -z "$petsc_sha256" || -z "$petsc_url" || -z "$petsc_mirror" ]]; then + echo "ERROR: cannot read the PETSc url, mirror and sha256 from [tool.proteus.modules.petsc]" \ + "in pyproject.toml; this needs python 3.11 or newer (tomllib) on PATH" >&2 + exit 1 +fi + # Clean previous installation rm -rf "$workpath" mkdir "$workpath" # ----------------------------------------------------------------------------- -# 3. Download PETSc 3.19.0 from OSF +# 3. Download the PETSc 3.19.0 archive: Zenodo first, then the DataverseNL mirror # ----------------------------------------------------------------------------- -current_step="Downloading PETSc archive from OSF" +current_step="Downloading PETSc archive" zipfile="$workpath/petsc.zip" -url="https://osf.io/download/p5vxq/" -echo "Downloading PETSc archive from OSF..." -echo " $url -> $zipfile" -sleep 1 -curl -LsS "$url" > "$zipfile" +fetch_verified "$petsc_sha256" "$zipfile" \ + "${PETSC_URL:-$petsc_url}" "${PETSC_MIRROR_URL:-$petsc_mirror}" current_step="Decompressing PETSc archive" echo "Decompressing..."