diff --git a/config/calibration/mask_v1.X.6_ppv1.yaml b/config/calibration/mask_v1.X.6_ppv1.yaml index bcaffb15..c27289a6 100644 --- a/config/calibration/mask_v1.X.6_ppv1.yaml +++ b/config/calibration/mask_v1.X.6_ppv1.yaml @@ -1,9 +1,11 @@ # Config file for masking and calibration. -# Standard cuts without halo masks, type v1.X.6. +# Joint ShapePipe + PhotoPipe catalogue. +# SP: Standard cuts without halo masks, type v1.X.6. +# PP" Standard selection. # General parameters (can also given on command line) params: - input_path: unions_shapepipe_comprehensive_struc_2024_v1.X.c.hdf5 + input_path: unions_shapepipe_comprehensive_struc_ugriz_2024_v1.X.c.hdf5 cmatrices: False sky_regions: False verbose: True @@ -99,12 +101,44 @@ dat: kind: equal value: False + # u-band footprint + - col_name: MASK_16_u + label: "u-band imaging" + kind: equal + value: False + + # g-band footprint + - col_name: MASK_32_g + label: "g-band imaging" + kind: equal + value: False + # r-band footprint - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False + # i-band footprint + - col_name: MASK_128_i + label: "i-band imaging" + kind: equal + value: False + + # z-band footprint, both z surveys: 256_z is WISHES (Subaru HSC), 2048_z2 is + # Pan-STARRS (the deeper data, not the shallow all-sky survey). Told apart by + # depth: median MAG_LIM_z = 25.54 vs MAG_LIM_z2 = 24.88. + # Each bit is only required where its own band has a magnitude: asking for + # both at once keeps only the overlap of the two surveys, which costs ~85% of + # the sample. Pairs with the MAG_GAAP_0p7_z / _z2 two-band cut in `dat`. + - col_name: MASK_256_z + col_name2: MASK_2048_z2 + mag_col: MAG_GAAP_0p7_z + mag_col2: MAG_GAAP_0p7_z2 + label: "z-band imaging" + kind: covered_2bands + value: -99 + # Maximask - col_name: MASK_1024_Maximask label: "maximask" @@ -135,3 +169,14 @@ metacal: # Weight for global response matrix, None for unweighted mean global_R_weight: w + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good" (observed) area + good: False + + # Output file path base + output_base: "footprint_binned" diff --git a/scripts/calibration/calibrate_comprehensive_cat.py b/scripts/calibration/calibrate_comprehensive_cat.py index c5248c3e..28dcb8c2 100644 --- a/scripts/calibration/calibrate_comprehensive_cat.py +++ b/scripts/calibration/calibrate_comprehensive_cat.py @@ -180,9 +180,21 @@ "FLUX_RADIUS", "MAGERR_AUTO", "NGMIX_T_NOSHEAR", - "NGMIX_T_PSF_RECONV_NOSHEAR", + "NGMIX_Tpsf_NOSHEAR", "fwhm_PSF", ] + +# PhotoPipe photo-z and multi-band photometry, when fill_photoz_bands.py has +# put them in the input. Selected against what the input actually holds, for +# two reasons: no catalogue carries every key (v1.6.c.3 has no Z_ML), and the +# image-simulation runs share this script with none of these columns at all. +photoz_cols = [key for key in cat.PHOTOZ_KEYS if key in set(dat.dtype.names or ())] +if photoz_cols: + print(f"Copying {len(photoz_cols)} PhotoPipe columns to the output catalogue") +else: + print("No PhotoPipe columns found in the input; none copied") +add_cols = add_cols + photoz_cols + add_cols_data = {} for key in add_cols: add_cols_data[key] = cat.get_col(dat, key, mask_combined._mask, mask_metacal) diff --git a/scripts/calibration/demo_apply_hsp_masks.py b/scripts/calibration/demo_apply_hsp_masks.py index e47c4d47..f5056a06 100644 --- a/scripts/calibration/demo_apply_hsp_masks.py +++ b/scripts/calibration/demo_apply_hsp_masks.py @@ -22,7 +22,6 @@ # + import os -import re import tracemalloc from sp_validation import catalog_builders as sp_joint @@ -39,41 +38,48 @@ obj = sp_joint.ApplyHspMasks() -# Bits set for non-tomographic catalogues -bit_list = [1, 2, 4, 8, 64, 1024] - # + +# Set parameters +base = "unions_shapepipe_comprehensive" + +# Major version +ver_maj = "v1.6" + +# Mask release, subdirectory of the mask directory +mask_release = "dr6-2026-08" + +year = 2022 if ver_maj == "v1.3" else 2024 + +# $HOME/masks is a link to the shared UNIONS mask directory +mask_base = f"{os.environ['HOME']}/masks" + +if ver_maj in ("v1.5", "v1.6"): + # Multi-band (tomographic) + bit_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 1024, 2048] + obj._params["mask_dir"] = f"{mask_base}/{mask_release}" + obj._params["file_base"] = "mask_ugriz_" +else: + # r-band (non-tomographic) + bit_list = [1, 2, 4, 8, 64, 1024] + obj._params["mask_dir"] = mask_base + obj._params["file_base"] = "mask_r_" + # combine bit list with & operator # equivalent to bits = sum(bit_list) # if elements in bit_list are of type 2^n - bits = 0 for b in bit_list: bits = bits | b print(bits) -# + -# Set parameters -base = "unions_shapepipe_comprehensive" -ver = "v1.3.c" - -if ver == "v1.3.c": - year = 2022 -else: - year = 2024 -ver_maj = "v1.5" - obj._params["input_path"] = f"{base}_{year}_{ver_maj}.c.hdf5" obj._params["output_path"] = f"{base}_struc_{year}_{ver_maj}.c.hdf5" -obj._params["mask_dir"] = f"{os.environ['HOME']}/{ver_maj}.x/masks" obj._params["nside"] = 131072 -obj._params["file_base"] = "mask_r_" obj._params["bits"] = bits -obj._params["aux_mask_files"] = f"{obj._params['mask_dir']}/coverage_{ver_maj}.x.hsp" -ver_x = re.sub(r"c$", "x", ver) -obj._params["aux_mask_files"] = f"{obj._params['mask_dir']}/coverage_{ver_x}.hsp" +# Coverage map is in the mask base directory, not in the mask release one +obj._params["aux_mask_files"] = f"{mask_base}/coverage_{ver_maj}.x.hsp" obj._params["aux_mask_labels"] = "npoint3" obj._params["verbose"] = True # - diff --git a/scripts/examples/create_binned_mask_comprehensive.py b/scripts/examples/create_binned_mask_comprehensive.py index 03546bce..fe0482f6 100644 --- a/scripts/examples/create_binned_mask_comprehensive.py +++ b/scripts/examples/create_binned_mask_comprehensive.py @@ -32,7 +32,7 @@ # %% # Read configuration file and set parameters -config = obj.read_config_set_params("config_for_compr_mask.yaml") +config = obj.read_config_set_params("config_mask.yaml") # %% # Get data. Set load_into_memory to False for very large files diff --git a/scripts/fill_photoz_bands.py b/scripts/fill_photoz_bands.py index 36a0a1b2..70d64d0e 100755 --- a/scripts/fill_photoz_bands.py +++ b/scripts/fill_photoz_bands.py @@ -38,6 +38,8 @@ from cs_util import args as cs_args from cs_util import logging +import sp_validation.catalog as sp_cat + FITS_HDU = 1 EMPTY_VALUE = -199 COPY_CHUNK = 2_000_000 # rows per chunk when copying input → output @@ -59,86 +61,9 @@ ("X_WORLD", "Y_WORLD"), ] -REQUESTED_KEYS = [ - "Z_B", - "Z_B_MIN", - "Z_B_MAX", - "T_B", - "Z_ML", - "MAG_GAAP_u", - "MAGERR_GAAP_u", - "MAG_GAAP_0p7_u", - "MAGERR_GAAP_0p7_u", - "MAG_GAAP_1p0_u", - "MAGERR_GAAP_1p0_u", - "FLAG_GAAP_u", - "MAG_LIM_u", - "FLUX_GAAP_u", - "FLUXERR_GAAP_u", - "EXTINCTION_u", - "MAG_GAAP_g", - "MAGERR_GAAP_g", - "MAG_GAAP_0p7_g", - "MAGERR_GAAP_0p7_g", - "MAG_GAAP_1p0_g", - "MAGERR_GAAP_1p0_g", - "FLAG_GAAP_g", - "MAG_LIM_g", - "FLUX_GAAP_g", - "FLUXERR_GAAP_g", - "EXTINCTION_g", - "MAG_GAAP_r", - "MAGERR_GAAP_r", - "MAG_GAAP_0p7_r", - "MAGERR_GAAP_0p7_r", - "MAG_GAAP_1p0_r", - "MAGERR_GAAP_1p0_r", - "FLAG_GAAP_r", - "MAG_LIM_r", - "FLUX_GAAP_r", - "FLUXERR_GAAP_r", - "EXTINCTION_r", - "MAG_GAAP_i", - "MAGERR_GAAP_i", - "MAG_GAAP_0p7_i", - "MAGERR_GAAP_0p7_i", - "MAG_GAAP_1p0_i", - "MAGERR_GAAP_1p0_i", - "FLAG_GAAP_i", - "MAG_LIM_i", - "FLUX_GAAP_i", - "FLUXERR_GAAP_i", - "EXTINCTION_i", - "MAG_GAAP_z", - "MAGERR_GAAP_z", - "MAG_GAAP_0p7_z", - "MAGERR_GAAP_0p7_z", - "MAG_GAAP_1p0_z", - "MAGERR_GAAP_1p0_z", - "FLAG_GAAP_z", - "MAG_LIM_z", - "FLUX_GAAP_z", - "FLUXERR_GAAP_z", - "EXTINCTION_z", - "MAG_GAAP_z2", - "MAGERR_GAAP_z2", - "MAG_GAAP_0p7_z2", - "MAGERR_GAAP_0p7_z2", - "MAG_GAAP_1p0_z2", - "MAGERR_GAAP_1p0_z2", - "FLAG_GAAP_z2", - "MAG_LIM_z2", - "FLUX_GAAP_z2", - "FLUXERR_GAAP_z2", - "EXTINCTION_z2", - "EXTINCTION", - "ODDS", - "CHI_SQUARED_BPZ", - "M_0", - "BPZ_FILT", - "BPZ_NONDETFILT", - "BPZ_FLAGFILT", -] +# The one list, shared with the calibration script so the two cannot +# drift (see sp_validation.catalog.PHOTOZ_KEYS). +REQUESTED_KEYS = sp_cat.PHOTOZ_KEYS def params_default(): @@ -681,7 +606,9 @@ def main(argv=None): ) as pbar: for start in range(0, n_total, SCAN_CHUNK): end = min(start + SCAN_CHUNK, n_total) - tile_chunk = dset[start:end]["TILE_ID"] + # .fields() reads only TILE_ID; dset[start:end]["TILE_ID"] + # would materialise all columns first (~3 GB per chunk) + tile_chunk = dset.fields("TILE_ID")[start:end] for local_i, tid in enumerate(tile_chunk): tile_index_map_lists[tid].append(start + local_i) pbar.update(end - start) diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index b594b9b6..df2ec3f3 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -185,8 +185,13 @@ def fill_cat_gal(cat_gal, dat, g_uncorr, gal_metacal, mask1, mask2, purpose="wei cat_gal["R_g22"] = gal_metacal.R22 cat_gal["NGMIX_T_NOSHEAR"] = sp_cat.get_col(dat, "NGMIX_T_NOSHEAR", mask1, mask2) + # v1.6 writes NGMIX_Tpsf_NOSHEAR, ShapePipe v2 NGMIX_T_PSF_RECONV_NOSHEAR; + # the output column keeps one name either way. cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] = sp_cat.get_col( - dat, "NGMIX_T_PSF_RECONV_NOSHEAR", mask1, mask2 + dat, + first_present(dat, "NGMIX_T_PSF_RECONV_NOSHEAR", "NGMIX_Tpsf_NOSHEAR"), + mask1, + mask2, ) cat_gal["size_ratio"] = ( cat_gal["NGMIX_T_NOSHEAR"] / cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] @@ -690,6 +695,27 @@ def get_calibrate_no_leakage_e_from_cat(path_cat_gal, weight_type="des", verbose return e1_noleak, e2_noleak +def first_present(data, *candidates): + """Return the first of `candidates` present in `data`. + + ngmix columns come in two grammars: v1.6 and earlier catalogues use + NGMIX_ELL__ and NGMIX_Tpsf_, ShapePipe v2 writes + NGMIX_G_ and NGMIX_T_PSF_RECONV_. + """ + names = getattr(getattr(data, "dtype", None), "names", None) + if names is None: + names = getattr(data, "colnames", None) + if names is None: + names = list(data.keys()) + for candidate in candidates: + if candidate in names: + return candidate + raise KeyError( + f"none of {list(candidates)} in the catalogue; it has neither ngmix" + f" column grammar" + ) + + class metacal: """Metacal. @@ -804,7 +830,7 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): dict_tmp["flag"] = masked_data[f"NGMIX_FLAGS_{name_shear}"] - # Ellipticity in named scalar components (ShapePipe-v2 grammar) + # Either ngmix grammar (see first_present) for comp in (0, 1): dict_tmp[f"g{comp + 1}"] = masked_data[ f"NGMIX_G{comp + 1}_{name_shear}" diff --git a/src/sp_validation/catalog.py b/src/sp_validation/catalog.py index b590993f..ec073dcb 100644 --- a/src/sp_validation/catalog.py +++ b/src/sp_validation/catalog.py @@ -24,6 +24,94 @@ from sp_validation import format, io from sp_validation.version import __version__ +# PhotoPipe photo-z and multi-band photometry columns, as added to a +# comprehensive catalogue by scripts/fill_photoz_bands.py and copied on to the +# cut catalogue by scripts/calibration/calibrate_comprehensive_cat.py. ONE list +# for both: when it lived only in the filling script, the calibration script +# had no idea these columns existed and dropped all 77 of them from +# v1.6.6. Not every catalogue carries every key (v1.6.c.3 has no Z_ML), and +# image-sim catalogues carry none, so consumers select on what the input holds. +PHOTOZ_KEYS = [ + "Z_B", + "Z_B_MIN", + "Z_B_MAX", + "T_B", + "Z_ML", + "MAG_GAAP_u", + "MAGERR_GAAP_u", + "MAG_GAAP_0p7_u", + "MAGERR_GAAP_0p7_u", + "MAG_GAAP_1p0_u", + "MAGERR_GAAP_1p0_u", + "FLAG_GAAP_u", + "MAG_LIM_u", + "FLUX_GAAP_u", + "FLUXERR_GAAP_u", + "EXTINCTION_u", + "MAG_GAAP_g", + "MAGERR_GAAP_g", + "MAG_GAAP_0p7_g", + "MAGERR_GAAP_0p7_g", + "MAG_GAAP_1p0_g", + "MAGERR_GAAP_1p0_g", + "FLAG_GAAP_g", + "MAG_LIM_g", + "FLUX_GAAP_g", + "FLUXERR_GAAP_g", + "EXTINCTION_g", + "MAG_GAAP_r", + "MAGERR_GAAP_r", + "MAG_GAAP_0p7_r", + "MAGERR_GAAP_0p7_r", + "MAG_GAAP_1p0_r", + "MAGERR_GAAP_1p0_r", + "FLAG_GAAP_r", + "MAG_LIM_r", + "FLUX_GAAP_r", + "FLUXERR_GAAP_r", + "EXTINCTION_r", + "MAG_GAAP_i", + "MAGERR_GAAP_i", + "MAG_GAAP_0p7_i", + "MAGERR_GAAP_0p7_i", + "MAG_GAAP_1p0_i", + "MAGERR_GAAP_1p0_i", + "FLAG_GAAP_i", + "MAG_LIM_i", + "FLUX_GAAP_i", + "FLUXERR_GAAP_i", + "EXTINCTION_i", + "MAG_GAAP_z", + "MAGERR_GAAP_z", + "MAG_GAAP_0p7_z", + "MAGERR_GAAP_0p7_z", + "MAG_GAAP_1p0_z", + "MAGERR_GAAP_1p0_z", + "FLAG_GAAP_z", + "MAG_LIM_z", + "FLUX_GAAP_z", + "FLUXERR_GAAP_z", + "EXTINCTION_z", + "MAG_GAAP_z2", + "MAGERR_GAAP_z2", + "MAG_GAAP_0p7_z2", + "MAGERR_GAAP_0p7_z2", + "MAG_GAAP_1p0_z2", + "MAGERR_GAAP_1p0_z2", + "FLAG_GAAP_z2", + "MAG_LIM_z2", + "FLUX_GAAP_z2", + "FLUXERR_GAAP_z2", + "EXTINCTION_z2", + "EXTINCTION", + "ODDS", + "CHI_SQUARED_BPZ", + "M_0", + "BPZ_FILT", + "BPZ_NONDETFILT", + "BPZ_FLAGFILT", +] + def print_mean_ellipticity( dd, diff --git a/src/sp_validation/masks.py b/src/sp_validation/masks.py index 37514508..bc40a76c 100644 --- a/src/sp_validation/masks.py +++ b/src/sp_validation/masks.py @@ -118,14 +118,22 @@ class Mask: kind : str operation type; see :func:`apply_condition` for the allowed values, plus "not_equal_2bands", which keeps objects whose value differs - from ``value`` in ``col_name`` *or* ``col_name2`` (two-band OR) + from ``value`` in ``col_name`` *or* ``col_name2`` (two-band OR), and + "covered_2bands", which keeps objects that have one band both + measured and inside that band's footprint (see :meth:`apply`) value : float or list value(s) to be used in mask operation col_name2 : str, optional - name of second column; required for (and only used by) kind - "not_equal_2bands" + name of second column; required for kinds "not_equal_2bands" and + "covered_2bands" + mag_col, mag_col2 : str, optional + magnitude column belonging to ``col_name`` and ``col_name2``; + required for (and only used by) kind "covered_2bands" dat : numpy.ndarray, optional input data, default is `None`; apply mask if given + dat_other : numpy.ndarray, optional + the catalogue's other group, holding ``mag_col``/``mag_col2`` when + those sit in a different group than the footprint bits verbose : bool, optional verbose output if ``True``; default is ``False`` @@ -138,7 +146,10 @@ def __init__( kind=None, value=0, col_name2=None, + mag_col=None, + mag_col2=None, dat=None, + dat_other=None, verbose=False, ): self._col_name = col_name @@ -147,7 +158,20 @@ def __init__( self._kind = kind if kind == "not_equal_2bands" and col_name2 is None: raise ValueError("kind 'not_equal_2bands' requires col_name2") + if kind == "covered_2bands": + if None in (col_name2, mag_col, mag_col2): + raise ValueError( + "kind 'covered_2bands' requires col_name2, mag_col and mag_col2" + ) + if dat_other is None: + raise ValueError( + "kind 'covered_2bands' requires dat_other: the magnitudes sit " + "in a different catalogue group than the footprint bits" + ) self._col_name2 = col_name2 + self._mag_col = mag_col + self._mag_col2 = mag_col2 + self._dat_other = dat_other self._num_ok = None self._verbose = verbose @@ -174,11 +198,28 @@ def from_list(cls, masks, label="combined", verbose=False): return my_mask + def _band_measured_and_covered(self, dat, bit_col, mag_col): + """One band: magnitude present, and the footprint bit says covered. + + The bit is set where the band has no imaging, so a covered position + has it ``False``. + """ + measured = apply_condition(self._dat_other[mag_col], "not_equal", self._value) + return measured & ~np.asarray(dat[bit_col], dtype=bool) + def apply(self, dat): if self._kind == "not_equal_2bands": self._mask = apply_condition( dat[self._col_name], "not_equal", self._value ) | apply_condition(dat[self._col_name2], "not_equal", self._value) + elif self._kind == "covered_2bands": + # Each footprint bit is only asked about where its own band + # supplies a magnitude, so requiring both bands at once -- which + # no object outside the overlap of the two surveys can satisfy -- + # is avoided. Keep an object with at least one usable band. + self._mask = self._band_measured_and_covered( + dat, self._col_name, self._mag_col + ) | self._band_measured_and_covered(dat, self._col_name2, self._mag_col2) else: self._mask = apply_condition(dat[self._col_name], self._kind, self._value) @@ -245,6 +286,14 @@ def print_condition(self, f_out, latex=False): file=f_out, ) + if self._kind == "covered_2bands": + print( + f"({self._mag_col} != {self._value} and {name} = False)" + f" or ({self._mag_col2} != {self._value}" + f" and {self._col_name2} = False)", + file=f_out, + ) + if self._kind == "range": print(f"{self._value[0]} {sign} {name} {sign} {self._value[1]}", file=f_out) @@ -264,12 +313,16 @@ def create_descr(self): """ sign = self.get_sign() - if sign is not None: - descr = f"{sign}{self._value}" + descr = f"{sign}{self._value}" if sign is not None else str(self._kind) if self._kind == "range": descr = f"{self._value[0]}<={self._col_name}<={self._value[1]}" if self._kind == "not_equal_2bands": descr = f"!={self._value} in {self._col_name} or {self._col_name2}" + if self._kind == "covered_2bands": + descr = ( + f"{self._mag_col}!={self._value} and {self._col_name}=False, or " + f"{self._mag_col2}!={self._value} and {self._col_name2}=False" + ) self._descr = descr # Create description for FITS header @@ -369,8 +422,9 @@ def get_masks_from_config(config, dat, masks_to_apply=None, verbose=False): ): raise ValueError(f"Range kind requires a list of two values, got {value}") - # Create mask instance and append to list - my_mask = Mask(**mask_params, dat=dat, verbose=verbose) + # Create mask instance and append to list; covered_2bands reads its + # magnitudes from the same table as the footprint bits + my_mask = Mask(**mask_params, dat=dat, dat_other=dat, verbose=verbose) labels[my_mask._col_name] = len(masks) masks.append(my_mask) diff --git a/src/sp_validation/tests/test_grammar_calibration.py b/src/sp_validation/tests/test_grammar_calibration.py index 9f247cbf..89ec7e9b 100644 --- a/src/sp_validation/tests/test_grammar_calibration.py +++ b/src/sp_validation/tests/test_grammar_calibration.py @@ -28,7 +28,9 @@ N = 4000 CONFIG_DIR = Path(__file__).resolve().parents[3] / "config" / "calibration" -BITS = (1, 2, 4, 8, 64, 1024) # the bits v1 post-processing wrote +# The bits v1 post-processing wrote; the ugriz catalogue adds 16, 32, 128, 256 +# and 2048 (u, g, i, z, z2 footprints). +BITS = (1, 2, 4, 8, 16, 32, 64, 128, 256, 1024, 2048) def _twins(): @@ -149,7 +151,14 @@ def test_mask_cut_is_identical(comprehensive): def test_config_selection_is_identical(comprehensive, config_name): config = yaml.safe_load((CONFIG_DIR / config_name).read_text()) available = set(comprehensive["v2"].dtype.names) - cuts = [cut for cut in config["dat"] if cut["col_name"] in available] + # A cut is testable when every column it reads is in the twin catalogue + # (two-band kinds also read col_name2 and magnitudes) + columns = ("col_name", "col_name2", "mag_col", "mag_col2") + cuts = [ + cut + for cut in config["dat"] + if all(cut[key] in available for key in columns if key in cut) + ] if not cuts: pytest.skip(f"{config_name} cuts on no column of the twin catalogue") config = {"dat": cuts} diff --git a/src/sp_validation/tests/test_masks.py b/src/sp_validation/tests/test_masks.py index 840ae259..2111b8e9 100644 --- a/src/sp_validation/tests/test_masks.py +++ b/src/sp_validation/tests/test_masks.py @@ -100,6 +100,55 @@ def test_not_equal_2bands_descr_names_both_columns(): assert my_mask._descr == "!=-99 in mag_z or mag_z2" +def test_covered_2bands_pairs_each_bit_with_its_own_band(): + + # Footprint bits (set = no imaging) in one group, magnitudes in the other + bits = np.array( + [(False, True), (False, True), (True, False), (True, True), (False, False)], + dtype=[("256_z", "?"), ("2048_z2", "?")], + ) + mags = np.array( + [(21.0, 20.0), (-99, 20.0), (21.0, 20.0), (21.0, 20.0), (-99, -99)], + dtype=[("mag_z", "f8"), ("mag_z2", "f8")], + ) + + my_mask = Mask( + "256_z", + "zband", + kind="covered_2bands", + value=-99, + col_name2="2048_z2", + mag_col="mag_z", + mag_col2="mag_z2", + dat=bits, + dat_other=mags, + ) + + # row 0: z measured and covered -> keep + # row 1: z absent, z2 measured but not covered -> drop + # row 2: z not covered, z2 measured and covered -> keep + # row 3: neither band covered -> drop + # row 4: both covered but neither measured -> drop + npt.assert_array_equal(my_mask._mask, [True, False, True, False, False]) + + +def test_covered_2bands_requires_its_columns_and_other_group(): + + with pytest.raises(ValueError, match="mag_col"): + Mask("256_z", "zband", kind="covered_2bands", value=-99, col_name2="2048_z2") + + with pytest.raises(ValueError, match="dat_other"): + Mask( + "256_z", + "zband", + kind="covered_2bands", + value=-99, + col_name2="2048_z2", + mag_col="mag_z", + mag_col2="mag_z2", + ) + + def test_dat_ext_cut_list_is_refused_not_dropped(): """A config's ``dat_ext`` cuts would be silently ignored; refuse them.""" config = { diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 39649717..e7fe20af 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -42,13 +42,14 @@ apptainer-args: >- # * ``slurm_account``: the slurm executor's guess fails on candide (its # ``sacct`` call errors) and warns on every launch. # -# * The excludes are the flaky nodes: n17 (mount issues), n36. +# * The excludes are the flaky nodes: n17 (mount issues), n36, n09; and +# n34, n35 (grid_3 image-sims jobs filled n35's /tmp on 2026-09-30). default-resources: slurm_account: "cusers" slurm_partition: "comp,pscomp" runtime: "60m" cpus_per_task: 12 - slurm_extra: "'--exclude=n17,n36'" + slurm_extra: "'--exclude=n09,n17,n34,n35,n36'" # Snakemake refuses a real run on a remote executor without a job bound. jobs: 100