From 69883589fe5353c54e4d1d4f63cd365525da178a Mon Sep 17 00:00:00 2001
From: timlichtenberg <lichtenberg.tim@gmail.com>
Date: Tue, 6 Oct 2026 21:05:56 +0200
Subject: [PATCH 1/5] Download PETSc from Zenodo, with the DataverseNL mirror
 as the fallback

get_petsc.sh fetches the PETSc 3.19.0 source archive from Zenodo 15805756 instead of OSF, falls back to the DataverseNL mirror when Zenodo fails, and accepts only a file with the pinned SHA-256. The sources and the hash are pinned in pyproject.toml [tool.proteus.modules.petsc]; the mirror field stays empty until the mirror is published, and PETSC_URL and PETSC_MIRROR_URL override the sources. The download helper fetch_verified lives in tools/_get_common.sh.
---
 docs/How-to/update_module_pins.md   |   5 +-
 pyproject.toml                      |   9 +--
 tests/tools/test_install_scripts.py | 101 ++++++++++++++++++++++++++++
 tools/_get_common.sh                |  22 ++++++
 tools/get_petsc.sh                  |  19 +++---
 5 files changed, 140 insertions(+), 16 deletions(-)

diff --git a/docs/How-to/update_module_pins.md b/docs/How-to/update_module_pins.md
index e89392389..d38555b97 100644
--- a/docs/How-to/update_module_pins.md
+++ b/docs/How-to/update_module_pins.md
@@ -23,8 +23,9 @@ module is distributed.
 
 
 A third entry, PETSc, is pinned in `[tool.proteus.modules.petsc]` by the SHA-256
-of a pre-built archive rather than a git ref, because it is downloaded as a
-binary from OSF.
+of its source archive rather than a git ref: `tools/get_petsc.sh` downloads it from
+Zenodo (`url`), or from the DataverseNL mirror (`mirror`) when Zenodo fails, and
+refuses a file with another SHA-256.
 
 **PyPI floors** are minimum bounds. When a user runs `pip install -e ".[develop]"`,
 pip resolves the newest release that satisfies every floor. The floor is the
diff --git a/pyproject.toml b/pyproject.toml
index 4de7d1591..c56b128f2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -328,10 +328,11 @@ url = "https://github.com/FormingWorlds/LavAtmos"
 ref = "c963dd2acf6c8a4c33d246d653342121666ae612"
 
 [tool.proteus.modules.petsc]
-# Pre-built PETSc archive on OSF. Required by SPIDER. tools/get_petsc.sh
-# downloads and validates by SHA-256.
-url = "https://osf.io/download/p5vxq/"
-sha256 = ""
+# PETSc 3.19.0 source archive (Zenodo 15805756), required by SPIDER. tools/get_petsc.sh
+# tries url, then the DataverseNL mirror (empty: no fallback), and checks the SHA-256.
+url = "https://zenodo.org/records/15805756/files/petsc.zip?download=1"
+mirror = ""
+sha256 = "c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d"
 
 [tool.proteus.coverage_fast]
 # Fast PR gate: unit-only coverage on PRs (smoke, integration, and slow tiers
diff --git a/tests/tools/test_install_scripts.py b/tests/tools/test_install_scripts.py
index 9345b04d8..10c19b337 100644
--- a/tests/tools/test_install_scripts.py
+++ b/tests/tools/test_install_scripts.py
@@ -635,6 +635,107 @@ def test_resolve_module_pin_reads_the_pin_and_stops_when_it_is_missing(tmp_path)
     assert 'REACHED_CLONE' not in missing.stdout
 
 
+def _sha256(data: bytes) -> str:
+    """Return the SHA-256 hex digest of ``data``."""
+    import hashlib
+
+    return hashlib.sha256(data).hexdigest()
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize('strict', [False, True])
+def test_fetch_verified_falls_back_to_the_next_source_with_the_pinned_hash(tmp_path, strict):
+    """A failed download, an empty url and a wrong file each give way to the next source.
+
+    The archive goes straight into ``unzip`` and a PETSc build, so only a file
+    with the pinned SHA-256 may end the search.
+    """
+    good, bad = tmp_path / 'good.zip', tmp_path / 'bad.zip'
+    good.write_bytes(b'petsc archive')
+    bad.write_bytes(b'an error page')
+    dest = tmp_path / 'petsc.zip'
+    body = (
+        f'fetch_verified {_sha256(b"petsc archive")} "{dest}" '
+        f'"file://{tmp_path}/missing.zip" "" "file://{bad}" "file://{good}"\n'
+        'echo REACHED_UNZIP\n'
+    )
+
+    result = _run_bash(_with_common(body, strict))
+
+    assert result.returncode == 0, result.stderr
+    assert dest.read_bytes() == b'petsc archive'
+    assert 'REACHED_UNZIP' in result.stdout
+    assert f'download from file://{tmp_path}/missing.zip failed' in result.stderr
+    assert (
+        f'file://{bad} served a file with SHA-256 {_sha256(b"an error page")}' in result.stderr
+    )
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize('strict', [False, True])
+def test_fetch_verified_stops_when_no_source_serves_the_pinned_file(tmp_path, strict):
+    """With no matching source the install stops and leaves no archive to unpack."""
+    bad = tmp_path / 'bad.zip'
+    bad.write_bytes(b'an error page')
+    dest = tmp_path / 'petsc.zip'
+    body = f'fetch_verified {"0" * 64} "{dest}" "file://{bad}" ""\necho REACHED_UNZIP\n'
+
+    result = _run_bash(_with_common(body, strict))
+
+    assert result.returncode == 1
+    assert 'REACHED_UNZIP' not in result.stdout
+    assert not dest.exists()
+    assert f'no source served petsc.zip with SHA-256 {"0" * 64}' in result.stderr
+
+
+@pytest.mark.unit
+def test_get_petsc_tries_zenodo_then_the_mirror_from_its_pins(tmp_path):
+    """The shipped download step passes the pinned hash and sources, overridable by
+    PETSC_URL and PETSC_MIRROR_URL, to fetch_verified in that order."""
+    fake_tools = tmp_path / 'tools'
+    fake_tools.mkdir()
+    shutil.copy2(COMMON_LIB, fake_tools / '_get_common.sh')
+    (fake_tools / '_module_pins.py').write_text('')
+    archive = tmp_path / 'mirror.zip'
+    archive.write_bytes(b'petsc archive')
+    stubs = tmp_path / 'stubs'
+    stubs.mkdir()
+    _write_stub(
+        stubs,
+        'python',
+        f'#!/bin/bash\ncase "$3" in sha256) echo {_sha256(b"petsc archive")} ;; '
+        'url) echo "file:///nowhere/petsc.zip" ;; mirror) echo "" ;; esac\n',
+    )
+    block = _extract_script_block(
+        'get_petsc.sh', 'zipfile="$workpath/petsc.zip"', 'current_step="Decompressing'
+    )
+    snippet = (
+        f'source "{fake_tools}/_get_common.sh"\nworkpath="{tmp_path}"\n{block}\necho DONE\n'
+    )
+    env = {**os.environ, 'PATH': f'{stubs}:{os.environ["PATH"]}'}
+
+    pinned = subprocess.run(['bash', '-c', snippet], capture_output=True, text=True, env=env)
+    assert pinned.returncode == 1
+    assert 'download from file:///nowhere/petsc.zip failed' in pinned.stderr
+
+    env['PETSC_MIRROR_URL'] = f'file://{archive}'
+    mirrored = subprocess.run(['bash', '-c', snippet], capture_output=True, text=True, env=env)
+    assert mirrored.returncode == 0, mirrored.stderr
+    assert (tmp_path / 'petsc.zip').read_bytes() == b'petsc archive'
+    assert 'DONE' in mirrored.stdout
+
+
+@pytest.mark.unit
+def test_pyproject_pins_the_petsc_archive_by_sha256():
+    """PETSc is pinned to its Zenodo archive by a SHA-256, with a mirror field."""
+    pin = tomllib.loads((TOOLS_DIR.parent / 'pyproject.toml').read_text())['tool']['proteus'][
+        'modules'
+    ]['petsc']
+    assert pin['url'] == 'https://zenodo.org/records/15805756/files/petsc.zip?download=1'
+    assert re.fullmatch(r'[0-9a-f]{64}', pin['sha256'])
+    assert 'mirror' in pin
+
+
 # ---------------------------------------------------------------------------
 # ERR trap tests
 # ---------------------------------------------------------------------------
diff --git a/tools/_get_common.sh b/tools/_get_common.sh
index 1425485ce..248eae008 100644
--- a/tools/_get_common.sh
+++ b/tools/_get_common.sh
@@ -170,3 +170,25 @@ resolve_module_pin() {
         exit 1
     fi
 }
+
+# Download <dest> from the first of the urls that serves a file with the given
+# SHA-256, skipping an empty url. Exits 1, with every url's failure named on
+# stderr, when none does.
+fetch_verified() {
+    local sha256="$1" dest="$2" url got
+    shift 2
+    for url in "$@"; do
+        [ -n "$url" ] || continue
+        echo "Downloading $url"
+        if curl -fLsS --retry 3 "$url" -o "$dest"; then
+            got=$( (shasum -a 256 "$dest" 2>/dev/null || sha256sum "$dest") | awk '{print $1}')
+            [ "$got" = "$sha256" ] && return 0
+            echo "WARNING: $url served a file with SHA-256 $got, not $sha256" >&2
+        else
+            echo "WARNING: download from $url failed" >&2
+        fi
+    done
+    rm -f "$dest"
+    echo "ERROR: no source served $(basename "$dest") with SHA-256 $sha256" >&2
+    exit 1
+}
diff --git a/tools/get_petsc.sh b/tools/get_petsc.sh
index 5a2e52ec8..da046279d 100755
--- a/tools/get_petsc.sh
+++ b/tools/get_petsc.sh
@@ -3,7 +3,8 @@
 # get_petsc.sh — Download, configure, and compile PETSc for PROTEUS/SPIDER
 # =============================================================================
 #
-# Downloads PETSc 3.19.0 from OSF and builds it with sundials2 support.
+# Downloads the PETSc 3.19.0 source archive from Zenodo, or its DataverseNL
+# mirror, checks its SHA-256, and builds it with sundials2 support.
 # SPIDER is a pure C code, so C++ and Fortran compilers are disabled.
 #
 # Supported platforms:
@@ -52,8 +53,8 @@ on_error() {
     case "$current_step" in
         *"Download"*)
             echo "   - Check your internet connection"
-            echo "   - Verify the OSF URL is accessible: $url"
-            echo "   - Try downloading manually: curl -LsS $url > petsc.zip"
+            echo "   - The sources and the SHA-256 are in pyproject.toml [tool.proteus.modules.petsc];"
+            echo "     PETSC_URL and PETSC_MIRROR_URL override the sources"
             ;;
         *"Decompress"*)
             echo "   - The downloaded archive may be corrupted"
@@ -122,16 +123,14 @@ rm -rf "$workpath"
 mkdir "$workpath"
 
 # -----------------------------------------------------------------------------
-# 3. Download PETSc 3.19.0 from OSF
+# 3. Download the PETSc 3.19.0 archive: Zenodo first, then the DataverseNL mirror
 # -----------------------------------------------------------------------------
-current_step="Downloading PETSc archive from OSF"
+current_step="Downloading PETSc archive"
 
 zipfile="$workpath/petsc.zip"
-url="https://osf.io/download/p5vxq/"
-echo "Downloading PETSc archive from OSF..."
-echo "    $url -> $zipfile"
-sleep 1
-curl -LsS "$url" > "$zipfile"
+petsc_pin() { python "$proteus_tools_dir/_module_pins.py" petsc "$1"; }
+fetch_verified "$(petsc_pin sha256)" "$zipfile" \
+    "${PETSC_URL:-$(petsc_pin url)}" "${PETSC_MIRROR_URL:-$(petsc_pin mirror)}"
 
 current_step="Decompressing PETSc archive"
 echo "Decompressing..."

From 961e9b07fa78717ea550e41d9b27c6352e61bab2 Mon Sep 17 00:00:00 2001
From: timlichtenberg <lichtenberg.tim@gmail.com>
Date: Wed, 7 Oct 2026 06:23:06 +0200
Subject: [PATCH 2/5] Point the PETSc download at its DataverseNL mirror

The PETSc archive is mirrored as DataverseNL 10.34894/BZDJPU (one file, petsc.zip, same bytes as Zenodo 15805756), so get_petsc.sh falls back to that file when the Zenodo download fails or its SHA-256 does not match.
---
 pyproject.toml                      | 4 ++--
 tests/tools/test_install_scripts.py | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index b374b2d5a..0f0088fc6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -324,9 +324,9 @@ ref = "c963dd2acf6c8a4c33d246d653342121666ae612"
 
 [tool.proteus.modules.petsc]
 # PETSc 3.19.0 source archive (Zenodo 15805756), required by SPIDER. tools/get_petsc.sh
-# tries url, then the DataverseNL mirror (empty: no fallback), and checks the SHA-256.
+# tries url, then the DataverseNL mirror (10.34894/BZDJPU), and checks the SHA-256.
 url = "https://zenodo.org/records/15805756/files/petsc.zip?download=1"
-mirror = ""
+mirror = "https://dataverse.nl/api/access/datafile/683669"
 sha256 = "c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d"
 
 [tool.proteus.coverage_fast]
diff --git a/tests/tools/test_install_scripts.py b/tests/tools/test_install_scripts.py
index 5a3ed1023..7212fcdd6 100644
--- a/tests/tools/test_install_scripts.py
+++ b/tests/tools/test_install_scripts.py
@@ -727,13 +727,13 @@ def test_get_petsc_tries_zenodo_then_the_mirror_from_its_pins(tmp_path):
 
 @pytest.mark.unit
 def test_pyproject_pins_the_petsc_archive_by_sha256():
-    """PETSc is pinned to its Zenodo archive by a SHA-256, with a mirror field."""
+    """PETSc is pinned to its Zenodo archive by a SHA-256, with the DataverseNL file as mirror."""
     pin = tomllib.loads((TOOLS_DIR.parent / 'pyproject.toml').read_text())['tool']['proteus'][
         'modules'
     ]['petsc']
     assert pin['url'] == 'https://zenodo.org/records/15805756/files/petsc.zip?download=1'
-    assert re.fullmatch(r'[0-9a-f]{64}', pin['sha256'])
-    assert 'mirror' in pin
+    assert pin['sha256'] == 'c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d'
+    assert pin['mirror'] == 'https://dataverse.nl/api/access/datafile/683669'
 
 
 # ---------------------------------------------------------------------------

From 91b4e550d77e351a163c8dd169351597ccaab194 Mon Sep 17 00:00:00 2001
From: timlichtenberg <lichtenberg.tim@gmail.com>
Date: Wed, 7 Oct 2026 08:16:15 +0200
Subject: [PATCH 3/5] Let a failed PETSc download reach the troubleshooting
 text

fetch_verified returns 1 instead of exiting, so the ERR trap of get_petsc.sh prints its troubleshooting text, including the PETSC_URL and PETSC_MIRROR_URL overrides. An empty SHA-256 pin stops before any download, and curl gives up on a connection or a stalled transfer so the next source is tried. The tests pin the Zenodo-then-mirror order, the overrides and the skip of an empty source; the optional-modules page describes the Zenodo download.
---
 docs/How-to/optionalmodules_installation.md |  3 +-
 tests/tools/test_install_scripts.py         | 63 ++++++++++++++++-----
 tools/_get_common.sh                        |  9 ++-
 tools/_module_pins.py                       |  2 +-
 4 files changed, 59 insertions(+), 18 deletions(-)

diff --git a/docs/How-to/optionalmodules_installation.md b/docs/How-to/optionalmodules_installation.md
index 6b4fdd821..6600b67c4 100644
--- a/docs/How-to/optionalmodules_installation.md
+++ b/docs/How-to/optionalmodules_installation.md
@@ -21,7 +21,8 @@ bash tools/get_spider.sh
 
 === "Linux"
 
-    The PETSc script downloads a pre-compiled version from OSF and configures
+    The PETSc script downloads the PETSc 3.19.0 source archive from Zenodo (or its
+    DataverseNL mirror), checks its SHA-256, builds it and configures
     `PETSC_DIR` and `PETSC_ARCH` automatically.
 
     !!! note "Fedora / RHEL"
diff --git a/tests/tools/test_install_scripts.py b/tests/tools/test_install_scripts.py
index 7212fcdd6..ba83361c9 100644
--- a/tests/tools/test_install_scripts.py
+++ b/tests/tools/test_install_scripts.py
@@ -674,18 +674,35 @@ def test_fetch_verified_falls_back_to_the_next_source_with_the_pinned_hash(tmp_p
 @pytest.mark.unit
 @pytest.mark.parametrize('strict', [False, True])
 def test_fetch_verified_stops_when_no_source_serves_the_pinned_file(tmp_path, strict):
-    """With no matching source the install stops and leaves no archive to unpack."""
+    """With no matching source it returns 1 and leaves no archive, so a get_* script's
+    ERR trap prints its troubleshooting text and the install stops before unzip."""
     bad = tmp_path / 'bad.zip'
     bad.write_bytes(b'an error page')
     dest = tmp_path / 'petsc.zip'
-    body = f'fetch_verified {"0" * 64} "{dest}" "file://{bad}" ""\necho REACHED_UNZIP\n'
+    call = f'fetch_verified {"0" * 64} "{dest}" "file://{bad}" ""'
 
-    result = _run_bash(_with_common(body, strict))
+    returned = _run_bash(_with_common(f'{call} || echo "RETURNED $?"\n', strict))
+    trapped = _run_bash(
+        _with_common(f'set -e\ntrap "echo TRAPPED" ERR\n{call}\necho REACHED_UNZIP\n', strict)
+    )
 
-    assert result.returncode == 1
-    assert 'REACHED_UNZIP' not in result.stdout
+    assert 'RETURNED 1' in returned.stdout
     assert not dest.exists()
-    assert f'no source served petsc.zip with SHA-256 {"0" * 64}' in result.stderr
+    assert f'no source served petsc.zip with SHA-256 {"0" * 64}' in returned.stderr
+    assert 'download from  failed' not in returned.stderr
+    assert trapped.returncode == 1
+    assert 'TRAPPED' in trapped.stdout and 'REACHED_UNZIP' not in trapped.stdout
+
+
+@pytest.mark.unit
+def test_fetch_verified_refuses_an_empty_hash(tmp_path):
+    """A pin that could not be read stops before any download."""
+    good = tmp_path / 'good.zip'
+    good.write_bytes(b'petsc archive')
+    result = _run_bash(_with_common(f'fetch_verified "" "{tmp_path}/p.zip" "file://{good}"\n'))
+    assert result.returncode == 1
+    assert 'no SHA-256 pin for p.zip' in result.stderr
+    assert 'Downloading' not in result.stdout
 
 
 @pytest.mark.unit
@@ -698,31 +715,49 @@ def test_get_petsc_tries_zenodo_then_the_mirror_from_its_pins(tmp_path):
     (fake_tools / '_module_pins.py').write_text('')
     archive = tmp_path / 'mirror.zip'
     archive.write_bytes(b'petsc archive')
+    wrong = tmp_path / 'wrong.zip'
+    wrong.write_bytes(b'an error page')
     stubs = tmp_path / 'stubs'
     stubs.mkdir()
     _write_stub(
         stubs,
         'python',
         f'#!/bin/bash\ncase "$3" in sha256) echo {_sha256(b"petsc archive")} ;; '
-        'url) echo "file:///nowhere/petsc.zip" ;; mirror) echo "" ;; esac\n',
+        f'url) echo "file://{wrong}" ;; mirror) echo "$PIN_MIRROR" ;; esac\n',
     )
     block = _extract_script_block(
         'get_petsc.sh', 'zipfile="$workpath/petsc.zip"', 'current_step="Decompressing'
     )
     snippet = (
-        f'source "{fake_tools}/_get_common.sh"\nworkpath="{tmp_path}"\n{block}\necho DONE\n'
+        f'set -e\nsource "{fake_tools}/_get_common.sh"\nworkpath="{tmp_path}"\n{block}\necho DONE\n'
     )
-    env = {**os.environ, 'PATH': f'{stubs}:{os.environ["PATH"]}'}
+    env = {**os.environ, 'PATH': f'{stubs}:{os.environ["PATH"]}', 'PIN_MIRROR': ''}
+
+    def run(**extra):
+        return subprocess.run(
+            ['bash', '-c', snippet], capture_output=True, text=True, env={**env, **extra}
+        )
 
-    pinned = subprocess.run(['bash', '-c', snippet], capture_output=True, text=True, env=env)
+    # No mirror pin: the empty source is skipped, not tried.
+    pinned = run()
     assert pinned.returncode == 1
-    assert 'download from file:///nowhere/petsc.zip failed' in pinned.stderr
+    assert f'file://{wrong} served a file' in pinned.stderr
+    assert 'download from  failed' not in pinned.stderr
 
-    env['PETSC_MIRROR_URL'] = f'file://{archive}'
-    mirrored = subprocess.run(['bash', '-c', snippet], capture_output=True, text=True, env=env)
+    # Zenodo first, then the mirror.
+    mirrored = run(PIN_MIRROR=f'file://{archive}')
     assert mirrored.returncode == 0, mirrored.stderr
+    out = mirrored.stdout
+    assert out.index(f'Downloading file://{wrong}') < out.index(f'Downloading file://{archive}')
     assert (tmp_path / 'petsc.zip').read_bytes() == b'petsc archive'
-    assert 'DONE' in mirrored.stdout
+    assert 'DONE' in out
+
+    # PETSC_URL replaces the pinned url, and PETSC_MIRROR_URL the pinned mirror.
+    overridden = run(PETSC_URL=f'file://{archive}', PIN_MIRROR=f'file://{wrong}')
+    assert overridden.returncode == 0, overridden.stderr
+    assert f'Downloading file://{wrong}' not in overridden.stdout
+    mirror_set = run(PETSC_MIRROR_URL=f'file://{archive}')
+    assert mirror_set.returncode == 0, mirror_set.stderr
 
 
 @pytest.mark.unit
diff --git a/tools/_get_common.sh b/tools/_get_common.sh
index 248eae008..b737e7e4a 100644
--- a/tools/_get_common.sh
+++ b/tools/_get_common.sh
@@ -177,10 +177,15 @@ resolve_module_pin() {
 fetch_verified() {
     local sha256="$1" dest="$2" url got
     shift 2
+    if [ -z "$sha256" ]; then
+        echo "ERROR: no SHA-256 pin for $(basename "$dest"); check tools/_module_pins.py" >&2
+        return 1
+    fi
     for url in "$@"; do
         [ -n "$url" ] || continue
         echo "Downloading $url"
-        if curl -fLsS --retry 3 "$url" -o "$dest"; then
+        if curl -fLsS --retry 3 --connect-timeout 30 --speed-limit 1024 --speed-time 60 \
+            "$url" -o "$dest"; then
             got=$( (shasum -a 256 "$dest" 2>/dev/null || sha256sum "$dest") | awk '{print $1}')
             [ "$got" = "$sha256" ] && return 0
             echo "WARNING: $url served a file with SHA-256 $got, not $sha256" >&2
@@ -190,5 +195,5 @@ fetch_verified() {
     done
     rm -f "$dest"
     echo "ERROR: no source served $(basename "$dest") with SHA-256 $sha256" >&2
-    exit 1
+    return 1
 }
diff --git a/tools/_module_pins.py b/tools/_module_pins.py
index 8ff79cbb1..70a7dd40f 100644
--- a/tools/_module_pins.py
+++ b/tools/_module_pins.py
@@ -2,7 +2,7 @@
 
 Single entry point for tools/get_*.sh and the CI composite action to look
 up the URL and ref of an external module (AGNI, SOCRATES, SPIDER, VULCAN,
-LovePy, PETSc). The source of truth is pyproject.toml's
+LovePy), or the url, mirror and sha256 of the PETSc archive. The source of truth is pyproject.toml's
 ``[tool.proteus.modules.<name>]`` table.
 
 Usage from a shell script:

From 47c0619185217f5234913a4c867591f5e54d6e29 Mon Sep 17 00:00:00 2001
From: timlichtenberg <lichtenberg.tim@gmail.com>
Date: Wed, 7 Oct 2026 09:11:58 +0200
Subject: [PATCH 4/5] Read the PETSc pins before removing the old installation

get_petsc.sh reads url, mirror and sha256 before it removes the previous installation and stops with the cause when one cannot be read, which needs python 3.11 or newer. The pin page gains a PETSc case (the three values change together, SPIDER holds its own copy, and the mirror can be checked alone with PETSC_URL broken). The test file is formatted with ruff 0.15.18 and its step strings name the current download.
---
 docs/How-to/update_module_pins.md   | 26 +++++++++++++++++++---
 tests/tools/test_install_scripts.py | 34 ++++++++++++++++++++---------
 tools/_module_pins.py               |  1 +
 tools/get_petsc.sh                  | 16 +++++++++++---
 4 files changed, 61 insertions(+), 16 deletions(-)

diff --git a/docs/How-to/update_module_pins.md b/docs/How-to/update_module_pins.md
index d38555b97..e819287ba 100644
--- a/docs/How-to/update_module_pins.md
+++ b/docs/How-to/update_module_pins.md
@@ -38,8 +38,8 @@ identical AGNI, SOCRATES, and SPIDER checkouts.
 
 ## How the update process works
 
-Everything downstream reads from the same table, so a pin only needs to be
-changed in one place:
+Everything in PROTEUS reads from the same table, so a pin is changed in
+`pyproject.toml` alone (PETSc is the exception, see Case 4):
 
 - **`tools/get_*.sh`** read git pins through `tools/_module_pins.py`
   (`python tools/_module_pins.py agni ref` prints the pinned AGNI commit) and
@@ -56,7 +56,8 @@ changed in one place:
   pinned but not drift-checked by the doctor.
 
 Because of this, bumping a module is a single-line edit followed by a badge
-refresh and a local re-sync.
+refresh and a local re-sync; a PETSc archive change is three lines here and the
+matching lines in SPIDER.
 
 ## Procedure
 
@@ -178,6 +179,25 @@ follow upstream, not a default.
     cannot diverge. Do not add a second pin for these in
     `[tool.proteus.modules]`.
 
+### Case 4: change the PETSc archive
+
+PETSc is pinned by its archive, not a ref: `url` (the Zenodo file), `mirror`
+(the DataverseNL file of the same archive) and `sha256` in
+`[tool.proteus.modules.petsc]` change together, since `tools/get_petsc.sh`
+accepts a file from either source only when its SHA-256 matches. SPIDER's
+`tools/get_petsc.sh` holds its own copy of the three values and is changed in the
+same window, so the two installs build the same PETSc.
+
+Check the mirror on its own by breaking the Zenodo source, so the download has
+to come from DataverseNL:
+
+```console
+PETSC_URL=file:///nonexistent/petsc.zip bash tools/get_petsc.sh /tmp/petsc-check
+```
+
+The log names the DataverseNL file it downloaded, and the build continues only if
+its SHA-256 matches the pin.
+
 ## Propagating the change to other developers
 
 Once the pin change is merged, other developers pick it up the same way a user
diff --git a/tests/tools/test_install_scripts.py b/tests/tools/test_install_scripts.py
index ba83361c9..71fc68b5b 100644
--- a/tests/tools/test_install_scripts.py
+++ b/tests/tools/test_install_scripts.py
@@ -726,10 +726,11 @@ def test_get_petsc_tries_zenodo_then_the_mirror_from_its_pins(tmp_path):
         f'url) echo "file://{wrong}" ;; mirror) echo "$PIN_MIRROR" ;; esac\n',
     )
     block = _extract_script_block(
-        'get_petsc.sh', 'zipfile="$workpath/petsc.zip"', 'current_step="Decompressing'
+        'get_petsc.sh', '# Read the archive pins', 'current_step="Decompressing'
     )
+    work = tmp_path / 'work'
     snippet = (
-        f'set -e\nsource "{fake_tools}/_get_common.sh"\nworkpath="{tmp_path}"\n{block}\necho DONE\n'
+        f'set -e\nsource "{fake_tools}/_get_common.sh"\nworkpath="{work}"\n{block}\necho DONE\n'
     )
     env = {**os.environ, 'PATH': f'{stubs}:{os.environ["PATH"]}', 'PIN_MIRROR': ''}
 
@@ -738,27 +739,40 @@ def run(**extra):
             ['bash', '-c', snippet], capture_output=True, text=True, env={**env, **extra}
         )
 
-    # No mirror pin: the empty source is skipped, not tried.
-    pinned = run()
+    # Both pinned sources serve a wrong file: the install stops.
+    pinned = run(PIN_MIRROR=f'file://{wrong}')
     assert pinned.returncode == 1
-    assert f'file://{wrong} served a file' in pinned.stderr
-    assert 'download from  failed' not in pinned.stderr
+    assert pinned.stderr.count(f'file://{wrong} served a file') == 2
+
+    # An empty pin stops with the cause instead of skipping that source.
+    empty_mirror = run()
+    assert empty_mirror.returncode == 1
+    assert 'cannot read the PETSc url, mirror and sha256' in empty_mirror.stderr
 
     # Zenodo first, then the mirror.
     mirrored = run(PIN_MIRROR=f'file://{archive}')
     assert mirrored.returncode == 0, mirrored.stderr
     out = mirrored.stdout
     assert out.index(f'Downloading file://{wrong}') < out.index(f'Downloading file://{archive}')
-    assert (tmp_path / 'petsc.zip').read_bytes() == b'petsc archive'
+    assert (work / 'petsc.zip').read_bytes() == b'petsc archive'
     assert 'DONE' in out
 
     # PETSC_URL replaces the pinned url, and PETSC_MIRROR_URL the pinned mirror.
     overridden = run(PETSC_URL=f'file://{archive}', PIN_MIRROR=f'file://{wrong}')
     assert overridden.returncode == 0, overridden.stderr
     assert f'Downloading file://{wrong}' not in overridden.stdout
-    mirror_set = run(PETSC_MIRROR_URL=f'file://{archive}')
+    mirror_set = run(PETSC_MIRROR_URL=f'file://{archive}', PIN_MIRROR=f'file://{wrong}')
     assert mirror_set.returncode == 0, mirror_set.stderr
 
+    # A pin that cannot be read stops before the previous installation is removed.
+    (work / 'old_build').write_text('kept')
+    _write_stub(stubs, 'python', '#!/bin/bash\nexit 1\n')
+    unreadable = run()
+    assert unreadable.returncode == 1
+    assert 'needs python 3.11 or newer (tomllib)' in unreadable.stderr
+    assert (work / 'old_build').read_text() == 'kept'
+    assert 'Downloading' not in unreadable.stdout
+
 
 @pytest.mark.unit
 def test_pyproject_pins_the_petsc_archive_by_sha256():
@@ -810,7 +824,7 @@ def test_err_trap_reports_current_step():
     """ERR trap output includes the ``current_step`` variable value."""
     snippet = """\
 set -e
-current_step="Downloading PETSc archive from OSF"
+current_step="Downloading PETSc archive"
 on_error() {
     local rc=$?
     echo "STEP=$current_step"
@@ -824,7 +838,7 @@ def test_err_trap_reports_current_step():
         text=True,
     )
     assert result.returncode != 0
-    assert 'STEP=Downloading PETSc archive from OSF' in result.stdout
+    assert 'STEP=Downloading PETSc archive' in result.stdout
 
 
 # ---------------------------------------------------------------------------
diff --git a/tools/_module_pins.py b/tools/_module_pins.py
index 70a7dd40f..6dd84f4f1 100644
--- a/tools/_module_pins.py
+++ b/tools/_module_pins.py
@@ -9,6 +9,7 @@
 
     URL=$(python tools/_module_pins.py agni url)
     REF=$(python tools/_module_pins.py agni ref)
+    MIRROR=$(python tools/_module_pins.py petsc mirror)
 
 Exits non-zero with a clear message if the module is unknown or the
 requested field is missing.
diff --git a/tools/get_petsc.sh b/tools/get_petsc.sh
index 6de396ba1..dc6cfc229 100755
--- a/tools/get_petsc.sh
+++ b/tools/get_petsc.sh
@@ -118,6 +118,17 @@ export PETSC_DIR="$workpath"
 echo "PETSC_DIR  = $PETSC_DIR"
 echo "PETSC_ARCH = $PETSC_ARCH"
 
+# Read the archive pins before the previous installation is removed
+petsc_pin() { python "$proteus_tools_dir/_module_pins.py" petsc "$1"; }
+petsc_sha256=$(petsc_pin sha256) || petsc_sha256=""
+petsc_url=$(petsc_pin url) || petsc_url=""
+petsc_mirror=$(petsc_pin mirror) || petsc_mirror=""
+if [[ -z "$petsc_sha256" || -z "$petsc_url" || -z "$petsc_mirror" ]]; then
+    echo "ERROR: cannot read the PETSc url, mirror and sha256 from [tool.proteus.modules.petsc]" \
+        "in pyproject.toml; this needs python 3.11 or newer (tomllib) on PATH" >&2
+    exit 1
+fi
+
 # Clean previous installation
 rm -rf "$workpath"
 mkdir "$workpath"
@@ -128,9 +139,8 @@ mkdir "$workpath"
 current_step="Downloading PETSc archive"
 
 zipfile="$workpath/petsc.zip"
-petsc_pin() { python "$proteus_tools_dir/_module_pins.py" petsc "$1"; }
-fetch_verified "$(petsc_pin sha256)" "$zipfile" \
-    "${PETSC_URL:-$(petsc_pin url)}" "${PETSC_MIRROR_URL:-$(petsc_pin mirror)}"
+fetch_verified "$petsc_sha256" "$zipfile" \
+    "${PETSC_URL:-$petsc_url}" "${PETSC_MIRROR_URL:-$petsc_mirror}"
 
 current_step="Decompressing PETSc archive"
 echo "Decompressing..."

From 81a458cf2e8c208ce2e3002ab7be58b4b7119c4f Mon Sep 17 00:00:00 2001
From: timlichtenberg <lichtenberg.tim@gmail.com>
Date: Wed, 7 Oct 2026 09:57:26 +0200
Subject: [PATCH 5/5] Stop the PETSc download when no hash tool is installed

fetch_verified stops before any download, naming shasum and sha256sum, when neither is installed, instead of failing inside the hash step; the test covers it next to the blank-pin case.
---
 tests/tools/test_install_scripts.py | 21 +++++++++++++++------
 tools/_get_common.sh                |  4 ++++
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/tests/tools/test_install_scripts.py b/tests/tools/test_install_scripts.py
index 71fc68b5b..c14088098 100644
--- a/tests/tools/test_install_scripts.py
+++ b/tests/tools/test_install_scripts.py
@@ -695,14 +695,23 @@ def test_fetch_verified_stops_when_no_source_serves_the_pinned_file(tmp_path, st
 
 
 @pytest.mark.unit
-def test_fetch_verified_refuses_an_empty_hash(tmp_path):
-    """A pin that could not be read stops before any download."""
+def test_fetch_verified_refuses_an_empty_hash_or_no_hash_tool(tmp_path):
+    """A pin that could not be read, or no shasum and no sha256sum, stops before any download."""
     good = tmp_path / 'good.zip'
     good.write_bytes(b'petsc archive')
-    result = _run_bash(_with_common(f'fetch_verified "" "{tmp_path}/p.zip" "file://{good}"\n'))
-    assert result.returncode == 1
-    assert 'no SHA-256 pin for p.zip' in result.stderr
-    assert 'Downloading' not in result.stdout
+    call = 'fetch_verified "{}" "' + f'{tmp_path}/p.zip" "file://{good}"\n'
+    blank = _run_bash(_with_common(call.format('')))
+    no_tool = subprocess.run(
+        ['/bin/bash', '-c', _with_common(call.format(_sha256(b'petsc archive')))],
+        capture_output=True,
+        text=True,
+        env={'PATH': str(tmp_path / 'empty')},
+    )
+    assert blank.returncode == 1 and 'no SHA-256 pin for p.zip' in blank.stderr
+    assert no_tool.returncode == 1
+    assert 'neither shasum nor sha256sum is installed' in no_tool.stderr
+    for result in (blank, no_tool):
+        assert 'Downloading' not in result.stdout
 
 
 @pytest.mark.unit
diff --git a/tools/_get_common.sh b/tools/_get_common.sh
index b737e7e4a..2058d488f 100644
--- a/tools/_get_common.sh
+++ b/tools/_get_common.sh
@@ -181,6 +181,10 @@ fetch_verified() {
         echo "ERROR: no SHA-256 pin for $(basename "$dest"); check tools/_module_pins.py" >&2
         return 1
     fi
+    if ! command -v shasum >/dev/null 2>&1 && ! command -v sha256sum >/dev/null 2>&1; then
+        echo "ERROR: neither shasum nor sha256sum is installed; install one of them" >&2
+        return 1
+    fi
     for url in "$@"; do
         [ -n "$url" ] || continue
         echo "Downloading $url"