Repository navigation
Photoz catalogue handling #409
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -180,9 +180,21 @@ | |||||
| "FLUX_RADIUS", | ||||||
| "MAGERR_AUTO", | ||||||
| "NGMIX_T_NOSHEAR", | ||||||
| "NGMIX_Tpsf_NOSHEAR", | ||||||
|
Collaborator
There was a problem hiding this comment.
Suggested change
(The |
||||||
| "fwhm_PSF", | ||||||
| ] | ||||||
|
|
||||||
| # PhotoPipe photo-z and multi-band photometry, when fill_photoz_bands.py has | ||||||
| # put them in the input. Selected against what the input actually holds, for | ||||||
| # two reasons: no catalogue carries every key (v1.6.c.3 has no Z_ML), and the | ||||||
| # image-simulation runs share this script with none of these columns at all. | ||||||
| photoz_cols = [key for key in cat.PHOTOZ_KEYS if key in set(dat.dtype.names or ())] | ||||||
| if photoz_cols: | ||||||
| print(f"Copying {len(photoz_cols)} PhotoPipe columns to the output catalogue") | ||||||
| else: | ||||||
| print("No PhotoPipe columns found in the input; none copied") | ||||||
| add_cols = add_cols + photoz_cols | ||||||
|
|
||||||
| add_cols_data = {} | ||||||
| for key in add_cols: | ||||||
| add_cols_data[key] = cat.get_col(dat, key, mask_combined._mask, mask_metacal) | ||||||
|
|
||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -22,7 +22,6 @@ | |
|
|
||
| # + | ||
| import os | ||
| import tracemalloc | ||
|
|
||
| from sp_validation import catalog_builders as sp_joint | ||
|
|
@@ -39,41 +38,48 @@ | |
| obj = sp_joint.ApplyHspMasks() | ||
|
|
||
|
|
||
| # + | ||
| # Set parameters | ||
| base = "unions_shapepipe_comprehensive" | ||
|
|
||
| # Major version | ||
| ver_maj = "v1.6" | ||
|
|
||
| # Mask release, subdirectory of the mask directory | ||
| mask_release = "dr6-2026-08" | ||
|
|
||
| year = 2022 if ver_maj == "v1.3" else 2024 | ||
|
|
||
| # $HOME/masks is a link to the shared UNIONS mask directory | ||
| mask_base = f"{os.environ['HOME']}/masks" | ||
|
|
||
| if ver_maj in ("v1.5", "v1.6"): | ||
| # Multi-band (tomographic) | ||
| bit_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 1024, 2048] | ||
| obj._params["mask_dir"] = f"{mask_base}/{mask_release}" | ||
| obj._params["file_base"] = "mask_ugriz_" | ||
| else: | ||
| # r-band (non-tomographic) | ||
| bit_list = [1, 2, 4, 8, 64, 1024] | ||
| obj._params["mask_dir"] = mask_base | ||
| obj._params["file_base"] = "mask_r_" | ||
|
|
||
| # combine bit list with & operator | ||
| # equivalent to bits = sum(bit_list) | ||
| # if elements in bit_list are of type 2^n | ||
| bits = 0 | ||
| for b in bit_list: | ||
| bits = bits | b | ||
| print(bits) | ||
|
|
||
| obj._params["input_path"] = f"{base}_{year}_{ver_maj}.c.hdf5" | ||
| obj._params["output_path"] = f"{base}_struc_{year}_{ver_maj}.c.hdf5" | ||
|
|
||
| obj._params["nside"] = 131072 | ||
| obj._params["bits"] = bits | ||
|
|
||
| # Coverage map is in the mask base directory, not in the mask release one | ||
| obj._params["aux_mask_files"] = f"{mask_base}/coverage_{ver_maj}.x.hsp" | ||
| obj._params["aux_mask_labels"] = "npoint3" | ||
| obj._params["verbose"] = True | ||
| # - | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -32,7 +32,7 @@ | |
|
|
||
| # %% | ||
| # Read configuration file and set parameters | ||
| config = obj.read_config_set_params("config_mask.yaml") | ||
|
|
||
| # %% | ||
| # Get data. Set load_into_memory to False for very large files | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -38,6 +38,8 @@ | |
| from cs_util import args as cs_args | ||
| from cs_util import logging | ||
|
|
||
| import sp_validation.catalog as sp_cat | ||
|
|
||
| FITS_HDU = 1 | ||
| EMPTY_VALUE = -199 | ||
| COPY_CHUNK = 2_000_000 # rows per chunk when copying input → output | ||
|
|
@@ -59,86 +61,9 @@ | |
| ("X_WORLD", "Y_WORLD"), | ||
| ] | ||
|
|
||
| # The one list, shared with the calibration script so the two cannot | ||
| # drift (see sp_validation.catalog.PHOTOZ_KEYS). | ||
| REQUESTED_KEYS = sp_cat.PHOTOZ_KEYS | ||
|
|
||
|
|
||
| def params_default(): | ||
|
|
@@ -681,7 +606,9 @@ def main(argv=None): | |
| ) as pbar: | ||
| for start in range(0, n_total, SCAN_CHUNK): | ||
| end = min(start + SCAN_CHUNK, n_total) | ||
| # .fields() reads only TILE_ID; dset[start:end]["TILE_ID"] | ||
| # would materialise all columns first (~3 GB per chunk) | ||
| tile_chunk = dset.fields("TILE_ID")[start:end] | ||
| for local_i, tid in enumerate(tile_chunk): | ||
| tile_index_map_lists[tid].append(start + local_i) | ||
| pbar.update(end - start) | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -185,8 +185,13 @@ def fill_cat_gal(cat_gal, dat, g_uncorr, gal_metacal, mask1, mask2, purpose="wei | |
| cat_gal["R_g22"] = gal_metacal.R22 | ||
|
|
||
| cat_gal["NGMIX_T_NOSHEAR"] = sp_cat.get_col(dat, "NGMIX_T_NOSHEAR", mask1, mask2) | ||
| # v1.6 writes NGMIX_Tpsf_NOSHEAR, ShapePipe v2 NGMIX_T_PSF_RECONV_NOSHEAR; | ||
| # the output column keeps one name either way. | ||
| cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] = sp_cat.get_col( | ||
| dat, | ||
| first_present(dat, "NGMIX_T_PSF_RECONV_NOSHEAR", "NGMIX_Tpsf_NOSHEAR"), | ||
| mask1, | ||
| mask2, | ||
| ) | ||
| cat_gal["size_ratio"] = ( | ||
| cat_gal["NGMIX_T_NOSHEAR"] / cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] | ||
|
|
@@ -690,6 +695,27 @@ def get_calibrate_no_leakage_e_from_cat(path_cat_gal, weight_type="des", verbose | |
| return e1_noleak, e2_noleak | ||
|
|
||
|
|
||
| def first_present(data, *candidates): | ||
| """Return the first of `candidates` present in `data`. | ||
|
|
||
| ngmix columns come in two grammars: v1.6 and earlier catalogues use | ||
| NGMIX_ELL_<shear>_<comp> and NGMIX_Tpsf_<shear>, ShapePipe v2 writes | ||
| NGMIX_G<n>_<shear> and NGMIX_T_PSF_RECONV_<shear>. | ||
| """ | ||
| names = getattr(getattr(data, "dtype", None), "names", None) | ||
| if names is None: | ||
| names = getattr(data, "colnames", None) | ||
| if names is None: | ||
| names = list(data.keys()) | ||
| for candidate in candidates: | ||
| if candidate in names: | ||
| return candidate | ||
| raise KeyError( | ||
| f"none of {list(candidates)} in the catalogue; it has neither ngmix" | ||
| f" column grammar" | ||
| ) | ||
|
|
||
|
|
||
| class metacal: | ||
| """Metacal. | ||
|
|
||
|
|
@@ -804,7 +830,7 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): | |
|
|
||
| dict_tmp["flag"] = masked_data[f"NGMIX_FLAGS_{name_shear}"] | ||
|
|
||
| # Either ngmix grammar (see first_present) | ||
| for comp in (0, 1): | ||
| dict_tmp[f"g{comp + 1}"] = masked_data[ | ||
| f"NGMIX_G{comp + 1}_{name_shear}" | ||
|
|
||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
MASK_256_zis inSPATIAL_CUTSinscripts/masking.py, so withfootprint_only=Truethis cut gets sent toapply_condition(), which only handles single-column cuts. The result isValueError: Unknown mask condition kind: 'covered_2bands'.It's worth deciding what footprint-only should mean for this cut. One option reproduces the catalogue selection, magnitude checks included. The other is purely geometric: the union of the two imaging footprints, with bad =
bad_z & bad_z2.demo_create_footprint_mask.pyhas the same gap, since it ignorescol_name2.