Make SETUP.sh reproduce CI, and stop the SCRIP fixture OOMing

SETUP.sh

@@ -2,30 +2,59 @@
 set -e
 
 echo "[SETUP] Checking prerequisites..."
-if ! python3 -c "import sys; assert sys.version_info >= (3, 11)" &> /dev/null; then
-  echo "[ERROR] Python 3.11 or newer is required."
-  exit 1
-fi
-
-# Not fatal. Local analysis works on any supported Python; only Globus Compute
-# submission is minor-version sensitive (globus/globus-compute#2139), so an
-# unsupported submitter is a warning for HPC users, not a blocked install.
-if ! python3 -c "import sys; assert sys.version_info[:2] == (3, 12)" &> /dev/null; then
-  echo "[WARN] Python 3.12 is the supported Globus Compute submitter."
-  echo "[WARN] Local execution is unaffected; HPC submission may hit WorkerLost."
-fi
-
-echo "[SETUP] Installing project dependencies with uv..."
 if ! command -v uv &> /dev/null; then
     echo "[INFO] Installing uv..."
     pip install uv
 fi
 
-uv sync
+# Deliberately *not* gated on the ambient `python3`. `.python-version` pins
+# 3.12 and uv provisions that interpreter itself, so the python3 on PATH is
+# not the one the project runs on. The old check aborted with "Python 3.11 or
+# newer is required" on any machine whose system python3 was older -- a
+# macOS box with 3.9 on PATH could not run this script at all, while
+# `uv sync` on the very same box builds a working 3.12 environment. It
+# refused the one interpreter that does not matter.
+echo "[SETUP] Installing project dependencies with uv..."
 
+# `--extra hpc --dev` matches the CI job that runs the hpc-marked tests.
+# A bare `uv sync` leaves globus-compute-sdk uninstalled, and the 25
+# hpc-marked tests then fail on `ModuleNotFoundError` rather than being
+# deselected -- the first thing a new contributor saw was a red suite
+# caused by the setup script itself. `--dev` brings in pytest, jsonschema
+# and mypy, which the suite asserts on rather than importing optionally.
+uv sync --extra hpc --dev
+
+# Checked after the sync, against the interpreter uv actually chose, which
+# is the one that would submit the Globus Compute task. Not fatal: local
+# analysis works on any supported Python, and only submission is
+# minor-version sensitive (globus/globus-compute#2139).
+if ! uv run python -c "import sys; assert sys.version_info[:2] == (3, 12)" &> /dev/null; then
+  echo "[WARN] Python 3.12 is the supported Globus Compute submitter;"
+  echo "[WARN] this environment is $(uv run python -c 'import sys; print("%d.%d" % sys.version_info[:2])')."
+  echo "[WARN] Local execution is unaffected; HPC submission may hit WorkerLost."
+fi
+
+# CI partitions the suite by marker in three jobs; this runs the two that
+# need no special build. Markers, not `--ignore`: a filename ignore has to
+# be repeated everywhere, says nothing about why, and hides the passing
+# tests in that file. `--ignore=tests/test_remote_agent.py` was hiding 16
+# tests that pass perfectly well.
+#
+# `yac` is left out because it needs a YAC build that no amount of pip can
+# provide (scripts/build_yac_local.sh), and `live` because it submits real
+# work to a configured Globus Compute endpoint.
 echo "[SETUP] Running automated tests (no external data required)..."
-uv run pytest tests/ --ignore=tests/test_remote_agent.py
+uv run pytest tests/ -m "not hpc and not live and not yac"
+
+# Separately, so that a missing or broken `hpc` extra is reported as the
+# distinct failure it is rather than disappearing into the main run.
+echo "[SETUP] Running HPC tests (mocked; no cluster required)..."
+uv run pytest tests/ -m "hpc and not yac"
 
+echo ""
+echo "[INFO] Not run here: tests marked 'yac' need a YAC build"
+echo "[INFO] (scripts/build_yac_local.sh); tests marked 'live' need a real"
+echo "[INFO] Globus Compute endpoint and the UXMCP_LIVE_* variables."
 echo ""
 echo "[SUCCESS] Review 'README.md' for local setup and 'docs/remote-hpc.md' for HPC bring-up."
 echo "[INFO] For HPC endpoints, run: uxarray-mcp setup && uxarray-mcp endpoints add <name> <uuid>"

tests/test_subset_bbox_lazy.py

@@ -61,13 +61,24 @@ def _write_oversized_scrip(path):
     gigabytes: the underlying data is all-zero uint8 with light
     compression, so the file is ~12 MB on disk and writes in a couple of
     seconds.
+
+    The chunking is what keeps that promise. ``chunks=(n_face, n_corner)``
+    is a single chunk of 2000 x 3,500,000 uint8 = 6.5 GiB per array, and
+    ``to_netcdf`` materializes a chunk at a time, so writing the two
+    variables needed ~7.5 GiB resident before the code under test ran at
+    all. That is under GitHub's 16 GiB runners and over a developer
+    laptop, so the fixture -- not the server -- was what failed locally.
+    Chunking along ``grid_corners`` holds each write to ~191 MiB and
+    leaves the test identical: the guard reads ``corner_var.size``, which
+    is the declared shape, and never touches the data.
     """
     import dask.array as da
 
     n_face, n_corner = 2000, 3_500_000
     shape = (n_face, n_corner)
-    lon = da.zeros(shape, chunks=(n_face, n_corner), dtype="u1")
-    lat = da.zeros(shape, chunks=(n_face, n_corner), dtype="u1")
+    chunks = (n_face, 100_000)
+    lon = da.zeros(shape, chunks=chunks, dtype="u1")
+    lat = da.zeros(shape, chunks=chunks, dtype="u1")
     ds = xr.Dataset(
         {
             "grid_corner_lon": (("grid_size", "grid_corners"), lon),

tests/test_suite_integrity.py

@@ -78,3 +78,64 @@ def test_the_suite_cannot_see_a_real_endpoint():
 
     assert config.endpoint_id is None
     assert config.endpoints == {}
+
+
+class TestSetupScriptMatchesCI:
+    """``SETUP.sh`` is the first thing a new contributor runs.
+
+    It drifted: it ran a bare ``uv sync`` and an unfiltered ``pytest``,
+    while the suite had moved to marker-based selection. Following it
+    produced 19 failures that were purely missing optional dependencies --
+    the script's own fault, on a tree where CI was green. These hold it to
+    the same partition CI uses so the same drift is caught here instead of
+    by someone's first afternoon.
+    """
+
+    @staticmethod
+    def _setup_text():
+        from pathlib import Path
+
+        return (Path(__file__).resolve().parent.parent / "SETUP.sh").read_text()
+
+    def test_it_installs_the_hpc_extra(self):
+        """Without it, the hpc-marked tests fail on ModuleNotFoundError.
+
+        They are mocked and need no cluster, so there is no reason for a
+        contributor not to run them.
+        """
+        assert "uv sync --extra hpc --dev" in self._setup_text()
+
+    def test_it_selects_by_marker_and_not_by_filename(self):
+        """``pytest.ini`` says CI selects by marker, never by filename.
+
+        The ``--ignore=tests/test_remote_agent.py`` that used to be here was
+        hiding 16 tests that pass.
+        """
+        # Only the commands, not the comments explaining why `--ignore` is
+        # the wrong tool -- those legitimately mention it by name.
+        commands = [
+            line
+            for line in self._setup_text().splitlines()
+            if "pytest" in line and not line.lstrip().startswith("#")
+        ]
+        assert commands, "SETUP.sh runs no tests at all"
+        for line in commands:
+            assert "--ignore" not in line, (
+                f"select by marker, not by filename: {line.strip()}"
+            )
+        joined = "\n".join(commands)
+        assert '-m "not hpc and not live and not yac"' in joined
+        assert '-m "hpc and not yac"' in joined
+
+    def test_it_does_not_gate_on_the_ambient_interpreter(self):
+        """``.python-version`` pins 3.12 and uv provisions it.
+
+        The old gate ran ``python3 -c 'assert sys.version_info >= (3, 11)'``
+        and exited 1, so a machine whose system python3 was 3.9 could not
+        run the script at all -- while ``uv sync`` on that same machine
+        builds a working 3.12 environment.
+        """
+        text = self._setup_text()
+        assert "Python 3.11 or newer is required" not in text, (
+            "the ambient python3 is not the interpreter the project runs on"
+        )