Wire the UNIONS external healsparse masks into the workflow (MASK_<flag value>_<name> columns)

astra.yaml

@@ -228,7 +228,7 @@ analyses:
         description: >-
           Exposure detection catalogues carrying IMAFLAGS_ISO (and MASK_EXT when
           maps are configured), the flag stamps ngmix reads, and the final
-          catalogue's optional MASK_<band> columns.
+          catalogue's MASK_<flag value>_<name> columns.
         inputs: [exposure_flags, sky_masks]
         decisions: [pixel_mask_source, psf_star_mask_veto, sky_mask_application]
     decisions:
@@ -299,21 +299,27 @@ analyses:
               Halos flag objects for the final catalogue; a star inside another
               star's halo is not thereby a bad PSF sample.
       sky_mask_application:
-        label: Object-level sky masking deferred downstream
+        label: Sky masks written as catalogue columns; any cut is downstream
         rationale: >-
-          The final catalogue ships every detected object. When MASK_EXT_PATHS
-          lists band:path pairs, make_cat queries each healsparse map at the
+          The final catalogue ships every detected object. MASK_EXT_PATHS lists
+          label:path pairs, and make_cat queries each healsparse map at the
           object's windowed position and writes the map value verbatim into a
-          MASK_<band> column; an object off coverage gets the map's sentinel
+          MASK_<label> column; an object off coverage gets the map's sentinel
           (False for boolean maps, reading as unmasked; typically -1 for
-          integer maps). The committed config sets no MASK_EXT_PATHS, so no
-          mask column is written and all mask cuts happen downstream.
-        default: deferred_downstream
+          integer maps). The committed config names the UNIONS band-combined
+          ugriz mask ladder, one boolean map per bit (nside 131072, True =
+          masked), labelled <flag value>_<name>: eleven columns,
+          MASK_1_Faint_star_halos to MASK_2048_z2, matched line for line in
+          final_cat.param. The flag value is the one in the map's file name;
+          the names are the ones sp_validation uses for the same bits.
+          ShapePipe makes no mask choice and cuts on none of them; which
+          columns to cut on is decided downstream, in sp_validation.
+        default: catalogue_columns
         options:
           deferred_downstream:
             label: No mask columns; all objects shipped
           catalogue_columns:
-            label: Per-band MASK_<band> columns from MASK_EXT_PATHS, no cut
+            label: Per-bit MASK_<label> columns from MASK_EXT_PATHS, no cut
           pipeline_cut:
             label: Drop masked objects inside the pipeline
             excluded: true

src/shapepipe/modules/make_cat_package/make_cat.py

@@ -191,6 +191,14 @@ def save_mask_ext_data(final_cat_file, band_paths, w_log):
     Query ``XWIN_WORLD`` and ``YWIN_WORLD`` in catalogue order and pass the
     ``query_map`` result to ``MASK_<BAND>`` unchanged.
 
+    @sc [decision:masking.sky_mask_application,label:convention] mask-ext-ladder-columns
+    The labels in ``MASK_EXT_PATHS`` are ``<flag value>_<name>`` for each map
+    of the UNIONS mask ladder (``1_Faint_star_halos`` ... ``2048_z2``), and
+    this function writes ``MASK_<label>`` verbatim. The eleven ``MASK_*``
+    lines in ``workflow/config/cfis/final_cat.param`` must match those labels
+    line for line: the post-processing merge fails every tile on a name this
+    function did not write. Nothing here cuts on any column.
+
     Parameters
     ----------
     final_cat_file : file_io.FITSCatalogue

src/shapepipe/pipeline/file_io.py

@@ -1601,7 +1601,7 @@ def _get_fits_col_type(self, col_data):
             col_type = "K"
         elif type(col_data[0]) in [float, np.float16, np.float32, np.float64]:
             col_type = "D"
-        elif type(col_data[0]) is bool:
+        elif type(col_data[0]) in [bool, np.bool_]:
             col_type = "L"
         elif type(col_data[0]) in [str, np.str_]:
             col_type = "A"

tests/module/test_make_cat_mask_ext.py

@@ -16,8 +16,8 @@
 
 healsparse = pytest.importorskip("healsparse")
 
-from shapepipe.modules.make_cat_package import make_cat
-from shapepipe.pipeline import file_io
+from shapepipe.modules.make_cat_package import make_cat  # noqa: E402
+from shapepipe.pipeline import file_io  # noqa: E402
 
 
 class _NullLogger:
@@ -49,7 +49,7 @@ def _make_map(value, dtype=np.int16, sentinel=-1):
 
 
 def _write_final_cat(path):
-    """Write a synthetic final_cat FITS with a RESULTS ext of known positions."""
+    """Write a final_cat FITS with a RESULTS extension of known positions."""
     data = np.empty(
         len(RA),
         dtype=[
@@ -127,3 +127,38 @@ def test_mask_ext_absent_is_noop(tmp_path):
 
     assert cols == {"NUMBER", "XWIN_WORLD", "YWIN_WORLD"}
     assert not any(name.startswith("MASK_") for name in cols)
+
+
+def test_boolean_halo_maps_preserve_bit_identity(tmp_path):
+    """Faint and bright halo columns preserve separate boolean maps."""
+    paths = {}
+    for label, values in (
+        ("2_Bright_star_halos", [False, True, True]),
+        ("1_Faint_star_halos", [True, False, True]),
+    ):
+        smap = healsparse.HealSparseMap.make_empty(
+            NSIDE_COVERAGE, NSIDE_SPARSE, np.bool_, sentinel=False
+        )
+        smap.update_values_pos(
+            RA[:3], DEC[:3], np.array(values, dtype=bool), lonlat=True
+        )
+        path = tmp_path / f"mask_{label}.hsp"
+        smap.write(str(path))
+        paths[label] = str(path)
+
+    cat_path = tmp_path / "final_cat-halos.fits"
+    _write_final_cat(cat_path)
+    cat = file_io.FITSCatalogue(
+        str(cat_path), open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite
+    )
+    make_cat.save_mask_ext_data(cat, paths, _NullLogger())
+
+    cat.open()
+    data = cat.get_data()
+    faint = data["MASK_1_Faint_star_halos"]
+    bright = data["MASK_2_Bright_star_halos"]
+    npt.assert_array_equal(faint, [True, False, True, False])
+    npt.assert_array_equal(bright, [False, True, True, False])
+    assert faint.dtype == np.dtype(bool)
+    assert bright.dtype == np.dtype(bool)
+    cat.close()

tests/unit/test_mask_ext_columns.py

@@ -0,0 +1,62 @@
+"""The MASK_* columns final_cat.param asks for are the ones make_cat writes.
+
+make_cat writes one ``MASK_<label>`` column per ``label:path`` pair in
+``MASK_EXT_PATHS`` (config_tile_Mc.ini); the post-processing merge reads
+``final_cat.param`` and fails every tile on a name make_cat did not write.
+Each label is ``<flag value>_<name>``, and the flag value is the one in the
+map's file name: a label on the wrong map would publish one mask under
+another's name. Enforces contract ``mask-ext-ladder-columns``.
+"""
+
+import configparser
+import os
+import re
+from pathlib import Path
+
+from shapepipe.modules.make_cat_package import make_cat
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+CONFIG_DIR = REPO_ROOT / "workflow" / "config" / "cfis"
+
+
+def _mask_ext_paths(monkeypatch):
+    monkeypatch.setenv("SP_INPUT_MASKS", "/dummy/masks")
+    config = configparser.ConfigParser(interpolation=None)
+    config.read(CONFIG_DIR / "config_tile_Mc.ini")
+    raw = config.get("MAKE_CAT_RUNNER", "MASK_EXT_PATHS")
+    band_paths = make_cat.parse_mask_ext_paths(os.path.expandvars(raw))
+    for path in band_paths.values():
+        assert path.startswith("/dummy/masks/"), path
+    return band_paths
+
+
+def _written_mask_columns(monkeypatch):
+    return [f"MASK_{label}" for label in _mask_ext_paths(monkeypatch)]
+
+
+def _requested_mask_columns():
+    lines = (CONFIG_DIR / "final_cat.param").read_text().splitlines()
+    return [line.strip() for line in lines if re.match(r"MASK_", line.strip())]
+
+
+def test_final_cat_mask_columns_match_mask_ext_paths(monkeypatch):
+    """final_cat.param's MASK_* lines are MASK_EXT_PATHS's labels, in order."""
+    written = _written_mask_columns(monkeypatch)
+    requested = _requested_mask_columns()
+    assert len(written) == 11, written
+    assert requested == written, (
+        "mask-ext-ladder-columns: final_cat.param's MASK_* lines "
+        f"{requested} != the columns MASK_EXT_PATHS makes make_cat write "
+        f"{written}"
+    )
+
+
+def test_mask_label_flag_value_is_the_maps(monkeypatch):
+    """Each label's flag value is the one in its map's file name."""
+    for label, path in _mask_ext_paths(monkeypatch).items():
+        value, _, name = label.partition("_")
+        assert name, f"label {label!r} is not <flag value>_<name>"
+        assert path.endswith(f"_n{value}.hsp"), (
+            f"mask-ext-ladder-columns: MASK_{label} is read from {path}"
+        )

tests/workflow/params_pin.json

@@ -4,40 +4,40 @@
     "all": "572122d8d1901e12ff591b30adf405f8920be8183129befececbb819f3392ed8",
     "clean_exposure": "22cb76b13a5205d20a02a9bd3b8c8bea5ea2801555e24dd2b84b11145f7e79d9",
     "clean_tile": "a5c07b0461526ed407df36a291deb866d4181524b4c8e0fbc3dd047fd9d28479",
-    "exp_get_images": "71e76ff7f96af1c5d2b85697cc5819e2271911f177a2075253f3b8cfa268c1a9",
+    "exp_get_images": "a9e9eca0a99a348e43e0cd44d0c0aae0f890773d03e35007fb77f886b7d21f5f",
     "exp_persist": "302e2837542bc1102430c27c81c600b7cda32e8bddcb5fd60d33950987609fff",
-    "exp_psf": "2c4f6d00f1939ccbf05b4982a0202a0ff92a4373a727f4e00f3aaab4eba03352",
-    "exp_split": "6e954f8f3d06f44d3f164675912ce27d6216648855d04968f9168bd7d0f2c4fa",
+    "exp_psf": "87b9fd799a54130eb3dd9c78bb5124838854c23f365df7a991a53f4c6ac4c3ff",
+    "exp_split": "8fc97dd48e76cd6c7fdf9ee388ba1c1a3dd63428cbe1961641fbebe76b05786e",
     "final_cat_merge": "e7f46859c4503a2220713d7bb2507555515d0a9632d780b20f14c59e32210023",
     "prepare_all_tiles": "b8f872a22adf014e25a7fa5198f49b71a6fe9e56042ed82b682bc8763970a844",
     "star_cat_merge": "6277450958474af5270982fa35360f2f237a29f7533c526ee9265dfd5acc07a0",
-    "tile_detect": "8336b148769e9d43b64f0d945c05e7d163c64dba4e6287e8c1964d1502e3ffe1",
+    "tile_detect": "1df9ca86e0a22fa65615d4efbcdc90f9bf6f1a7cf71c0a4b9d8ab56b811b6c58",
     "tile_exp_forest": "7447ab4a1049de5f0b5c81e5f9ed2a8644c7bdab85cb0a060bfde81261a89b28",
-    "tile_find_exposures": "8704317871744996c44351c2836fcb222d7986a602e9e046a90d684ba7b3c184",
-    "tile_get_catalogue": "7e3f889a955a14b0b917a015c2a8bc90e433a89b1cc5e00b8ca94695c4c93d3c",
-    "tile_get_images": "331a67e747f211ebf4c14b946a7af7f9fc9f55243d69d2791d74aecc3ca228c3",
-    "tile_make_cat": "ea546ec59bcd13c0f8c9ee2c7fa5dde4773975277eed8f46ab63e6d124db0535",
-    "tile_merge_cats": "ff21216ea804dccc2d2c290d2b2499d5d05f0c34c0a56993c233f43fe3c06bdb",
-    "tile_merge_headers": "7a344849d62936e2f5598dc8731a2c4947eff2c4c7218b1a731dd2a9577e7111",
-    "tile_ngmix": "5192e65a72b3b6186b29d7bbecfe48751c7bc9ec68a61083ef4092db9f86f02f",
-    "tile_uncompress": "1e2b01acbf9708e0371070fb01c5b9568d7efb5f89d1835fc6bf91e2c8b60cb3",
-    "tile_vignets": "ab52bf2c6ede04915c77f30a44be0cf707c4609ddf8ee768bfaf598cfff31481"
+    "tile_find_exposures": "c883495a7ec196c61ef770507865f43834b08523793d337e8ddeaef0ad62f117",
+    "tile_get_catalogue": "84ffd03c4bde248c06aa58a0f4b7f399390a0ff3ecbcf07e03d718df4fe0cd7f",
+    "tile_get_images": "96e235dad9cdcde8ecf11f4516bc2303dbba1c8115ad343cdb26d3d1513da671",
+    "tile_make_cat": "1558e38ee60bf9d1e794c303567442b1d298898ab8d56bf1b30aa63974d212dd",
+    "tile_merge_cats": "0783c93deeb7f90bdb9772bbc4117851dff1a671518c81f67edd4c7c79cc7e6a",
+    "tile_merge_headers": "3e674f961fd5bb93ce1f3b0e2bb072a25a421f2828315a3db45a960c44a42111",
+    "tile_ngmix": "87618cc2318729e5c09d8f2915361f79a3fd0773d3eedd93cb9f94dcb3d97d5b",
+    "tile_uncompress": "4a013cdd04bc3e3c1082e477161d2b03e0eefe280b1aabb9af25c149a977f55b",
+    "tile_vignets": "89d95bd9107c124e86dfb029d5ba2a2fc1871311a71a8fe18466302ef2213b14"
   },
   "schema": 1,
-  "sha256": "1c792868397e4a2f1144d68d153c4a0a23e9ffd07ecdb95b98896d1b990c8ce1",
+  "sha256": "eab6930e0fd1044b9ce7a99172dbf9cbdcdec21291f8fa640d6e60c2efd3dd03",
   "unit_pre": {
-    "exp_get_images": "8dec850af212879f225fcf27a5f1281e1a075264c7b97d38c2214395d360168c",
-    "exp_psf": "f2358ddf7385918dc5033d10b37f6dc97a15d02b071a3ea0a4619a5f7e6f5bec",
-    "exp_split": "358fa8bbe59680d4f9839e007b343dd25fed733cf3156dccd30d305d33ac9480",
-    "tile_detect": "adad5d671fa65dd04433e1c82a725b845635e9d2e368a1e70833ed990df2d55a",
-    "tile_find_exposures": "c5922fb507f6fd040a179b53fba0818697661016c9688984b6ce249186dc6986",
-    "tile_get_catalogue": "b124d12252617abb8df8a98d6234ddfd6cc503e19fa72c9a0c48c21559d6f48c",
-    "tile_get_images": "45b47c44bfeb34ac973b89d4e752c8c82e78028f9f0da379c44627005b45e279",
-    "tile_make_cat": "7579a52e32e76523c0b77d75c468a47d7f2e2ca0c55865f40e9628a7481cd79d",
-    "tile_merge_cats": "69cfe94ba941d2ba7b1ce24883961b47b191b80aff381da1865c6bc38de0c354",
-    "tile_merge_headers": "8590cfa3281c88d43eb8c4fe5760176a13e928a419cbea9bea3a8f88b0634b42",
-    "tile_ngmix": "b6b75b553a62df54aea6c337dcde1e0cdcd0fe30000e86880c1a4689e42c4a72",
-    "tile_uncompress": "91a15538491e53ee2b0d52b0472909374270918e4b4c80880c4f5a8329e40161",
-    "tile_vignets": "a6111cef708aa3c9fb0144b49de9f781eef84d2096ba4f8a3de9bb45b1a3774d"
+    "exp_get_images": "dac6685a207dae3ea81d296ce53e9ba75d4636cf1f81e6399f2dda452680c992",
+    "exp_psf": "87fc8ea1153a709ab7ba56542cfacc041f5f8b7f6cf2f38a9bc9d8004d1cecb4",
+    "exp_split": "922a62360969a7d8775172debdb77469ab8552d8031fc559d92d63fc30443122",
+    "tile_detect": "cdd19095ce9e33847284825ed013aa0a35011e9877a3d89d6b8dc0349c6b7707",
+    "tile_find_exposures": "89f0816fcfb513154aa81783cc361add768b0101b4cce19426bdf4b43db0cf71",
+    "tile_get_catalogue": "2030933b663be3d229d328ee6266eda5a63c98589cafa1151a203e88c8dc733a",
+    "tile_get_images": "482acaad46e677ec9d67105c48b5d56116c7dff2d77f4a62bda73061721a7103",
+    "tile_make_cat": "af03252466961e9613fe800e42dff32ff9a3f476726012a1fe8d1c18067ee95d",
+    "tile_merge_cats": "9c6cfbe27fa9618df2418953c056c8ad7cdf64d4b6b0d8a73a32168c39b82114",
+    "tile_merge_headers": "6bd3343d183e5413cedb6b82493445602cc139d4b56fde39281c59085a0efa8d",
+    "tile_ngmix": "47abe4514dc948fb2115d244853da1f792c9f7eea51a207f64de3ee563dad76c",
+    "tile_uncompress": "bf181cb759b56143c7f086f614c559d4cececa54ae7f4c55b78fd8bfad096ba5",
+    "tile_vignets": "348a047b4c2e22d0d68b819a1b0ac3c9677c1ee7914557ac257139290548127a"
   }
 }

universes/committed.yaml

@@ -9,7 +9,7 @@ analyses:
     decisions:
       pixel_mask_source: instrument_flags_only
       psf_star_mask_veto: instrument_flags_only
-      sky_mask_application: deferred_downstream
+      sky_mask_application: catalogue_columns
   detection:
     decisions:
       tile_detection: unions_catalogue

workflow/README.md

@@ -302,6 +302,17 @@ profiles/nibi/config.yaml  SLURM executor; apptainer SDM; per-user jobs cap; kee
   catalogue server, staged, or rasterized, which is why the old
   `star_catalogue` / `exp_star_cat` / `exp_mask` rules and their cache root are
   gone.
+- **External masks are wired, on the tile side only (data runs).** `inputs.masks`
+  is a third input root beside tiles and exposures, set per machine in the
+  `machines:` table, exported as `$SP_INPUT_MASKS` and
+  pointing at the UNIONS DR6 ugriz bit ladder: one boolean healsparse map per
+  bit, nside 131072, `True` = masked. `config_tile_Mc.ini` names all 11 of them
+  in `MASK_EXT_PATHS` under a `<flag value>_<name>` label, so `make_cat`
+  writes `MASK_1_Faint_star_halos` … `MASK_2048_z2` and `final_cat.param`
+  carries the matching 11 names; the config holds the label table.
+  `MASK_2048_z2` is True where there is *no* Pan-STARRS z data, so an OR over
+  every column masks everything. ShapePipe cuts on none of them; the mask
+  choice is sp_validation's.
 - **The index is parse-time data, never a rule input.** Appending tiles
   changes which jobs exist without invalidating completed work.
 - **Exposure products are not `temp()`.** Exposures overlap tiles, so

workflow/Snakefile

@@ -90,6 +90,15 @@ if _unresolved:
         f"$variable for machine={MACHINE!r}, "
         f"input_type={INPUT_TYPE!r}: {', '.join(_unresolved)}. Set them in "
         f"your run config (SP_RUN_CONFIG).")
+# The external masks are a data-only input: config_tile_Mc.ini's
+# MASK_EXT_PATHS reads $SP_INPUT_MASKS, and the sims Mc.ini carries no mask
+# block. Checked here rather than in run_config.REQUIRED because it depends on
+# input_type.
+if INPUT_TYPE == "data" and not config.get("inputs", {}).get("masks"):
+    raise WorkflowError(
+        f"inputs.masks is unset for machine={MACHINE!r}, input_type='data': "
+        "the tile chain queries the UNIONS sky-fixed masks (MASK_EXT_PATHS in "
+        "config/cfis/config_tile_Mc.ini). Set it in the run config.")
 
 # The image every job runs in: this user's sandbox, else their cached SIF,
 # else the run config's `container:`. Checked at parse time, since an empty
@@ -1139,10 +1148,10 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None,
     There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at
     the committed config dir.
 
-    It also exports the configured input roots as ``SP_INPUT_TILES`` and
-    ``SP_INPUT_EXPOSURES``, the retrieve mode as ``SP_RETRIEVE`` and the PSF
-    choice as ``SP_PSF`` for the committed ini chain (plus ``PSF_DICT`` for the
-    image-simulation true PSF).
+    It also exports the configured input roots as ``SP_INPUT_TILES``,
+    ``SP_INPUT_EXPOSURES`` and (data only) ``SP_INPUT_MASKS``, the retrieve
+    mode as ``SP_RETRIEVE`` and the PSF choice as ``SP_PSF`` for the committed
+    ini chain (plus ``PSF_DICT`` for the image-simulation true PSF).
 
     Every line here is part of each rule's ``params.pre`` and so of the
     ``params`` rerun trigger: a line added for every rule reruns every finished
@@ -1181,6 +1190,8 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None,
     ]
     if PSF_DICT:
         lines.append(f"export PSF_DICT='{PSF_DICT}'")
+    if INPUT_TYPE == "data":
+        lines.append(f"export SP_INPUT_MASKS='{INPUTS['masks']}'")
     if forest:
         lines.append(f"export SP_EXP='{forest}'")
     for k, v in (env or {}).items():

workflow/config.yaml

@@ -44,7 +44,12 @@ input_types:
 # - base_dir
 # - tile_list: text file with tile IDs
 # - retrieve: method to get input data, allowed are symlink, vos
-# - inputs.tiles,, .exposures: path for input tile and exposure
+# - inputs.tiles, .exposures: path for input tile and exposure
+# - inputs.masks (data only): the UNIONS sky-fixed masks, one boolean
+#   healsparse map per bit of the band-combined ugriz mask (nside 131072,
+#   True = masked), queried per object by make_cat -- never rasterized.
+#   Consumed by MASK_EXT_PATHS in config/cfis/config_tile_Mc.ini, which
+#   carries the bit table. Required for input_type: data.
 # - inputs.catalogues: UNIONS per-tile catalogues, a local directory
 #   (symlinked in) or a vos: URL (tile_detection: unions_catalogue)
 # - psf_dict: simulation PSF stamps (psf_model: fake)
@@ -63,6 +68,7 @@ machines:
       inputs:
         tiles: $base_dir/unions-wl/tiles
         exposures: $base_dir/unions-wl/exposures
+        masks: $base_dir/unions-wl/masks/dr6-2026-08
         # UNIONS per-tile catalogues (CFIS.<tile>.r.cat), fetched by vcp in
         # the job: needs network from compute nodes and ~/.ssl/cadcproxy.pem.
         catalogues: vos:cfis/tiles_DR6
@@ -77,6 +83,7 @@ machines:
     data:
       retrieve: vos
       inputs:
+        masks: $base_dir/masks/dr6-2026-08
         catalogues: vos:cfis/tiles_DR6
       container: /n17data/cdaley/containers/shapepipe_develop-runtime-20260718.sif
     image_sims: