From c90ae4ff5bab50936a63c29443d9176bfb684705 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Wed, 25 Feb 2026 16:23:46 +0100 Subject: [PATCH 01/14] additive bias calculation for paper updated to v1.4.6.3 --- notebooks/cosmo_val/cat_config.yaml | 276 ++++++++++++++++++++++++++++ pyproject.toml | 4 + 2 files changed, 280 insertions(+) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index a1a6b756..6596de22 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -726,6 +726,7 @@ SP_v1.4.8: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits +<<<<<<< Updated upstream # For additive bias only SP_v1.4.6_uncal: @@ -830,6 +831,281 @@ SP_v1.4.8_uncal: hdu: 1 patch_number: 100 +======= +SP_v1.4.11.2: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: orange + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: s + cov_th: + A: 2894.0303815287743 + n_e: 6.25812596800506 + n_psf: 0.752316232272063 + sigma_e: 0.380499698062307 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: /n17data/UNIONS/WL/v1.4.x/v1.4.11.2/unions_shapepipe_cut_struc_2024_v1.4.11.2.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.11.3: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: red + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: v + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.12.3: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: lightblue + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: ^ + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.12.3/unions_shapepipe_cut_struc_2024_v1.4.12.3.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.13.3: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: cyan + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: h + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.13.3/unions_shapepipe_cut_struc_2024_v1.4.13.3.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.11.3_ecut07: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: darkblue + getdist_colour: 0.0, 0.3, 0.7 + ls: dotted + marker: s + cov_th: + A: 2894.0303815287743 + n_e: 5.830114732534133 + n_psf: 0.752316232272063 + sigma_e: 0.34058513153783426 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: /n17data/cdaley/unions/pure_eb/results/ecut/SP_v1.4.11.3_ecut07.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.6.3_uncal: + pipeline: SP + subdir: /n17data/UNIONS/WL/v1.4.x + shear: + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + ra_col: RA + dec_col: Dec + e1_col: e1_uncal + e2_col: e2_uncal + w_col: w_des + R: 1.0 + psf: + path: unions_shapepipe_psf_2024_v1.4.a.fits + hdu: 1 + patch_number: 100 +SP_v1.4.6.3_uncal_w_iv: + pipeline: SP + subdir: /n17data/UNIONS/WL/v1.4.x + shear: + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + ra_col: RA + dec_col: Dec + e1_col: e1_uncal + e2_col: e2_uncal + w_col: w_iv + R: 1.0 + psf: + path: unions_shapepipe_psf_2024_v1.4.a.fits + hdu: 1 + patch_number: 100 +SP_v1.4.6.3_uncal_w_1: + pipeline: SP + subdir: /n17data/UNIONS/WL/v1.4.x + shear: + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + ra_col: RA + dec_col: Dec + e1_col: e1_uncal + e2_col: e2_uncal + w_col: one + R: 1.0 + psf: + path: unions_shapepipe_psf_2024_v1.4.a.fits + hdu: 1 + patch_number: 100 +>>>>>>> Stashed changes SP_v1.4_LFmask_8k: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask pipeline: SP diff --git a/pyproject.toml b/pyproject.toml index d0ee1d58..f706548b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,11 @@ dependencies = [ "lenspack", "lmfit", "numexpr", +<<<<<<< Updated upstream "numpy>=1.14", +======= + "numpy>=1.26", +>>>>>>> Stashed changes "opencv-python-headless", "pyccl", "pyarrow", From 392461e7eb3d79f3c93849aad74aed12bb6f3b40 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 20 Apr 2026 11:23:30 +0200 Subject: [PATCH 02/14] added config --- notebooks/cosmo_val/cat_config.yaml | 69 +++-------------------------- 1 file changed, 6 insertions(+), 63 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 6596de22..2c95f9f8 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -726,14 +726,11 @@ SP_v1.4.8: e1_col: e1 e2_col: e2 path: unions_shapepipe_star_2024_v1.4.a.fits -<<<<<<< Updated upstream - -# For additive bias only -SP_v1.4.6_uncal: +SP_v1.4.6.3_uncal: pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: - path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec @@ -746,11 +743,11 @@ SP_v1.4.6_uncal: hdu: 1 patch_number: 100 -SP_v1.4.6_uncal_w_iv: +SP_v1.4.6.3_uncal_w_iv: pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: - path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec @@ -763,11 +760,11 @@ SP_v1.4.6_uncal_w_iv: hdu: 1 patch_number: 100 -SP_v1.4.6_uncal_w_1: +SP_v1.4.6.3_uncal_w_1: pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: - path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec @@ -779,59 +776,6 @@ SP_v1.4.6_uncal_w_1: path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 patch_number: 100 - -SP_v1.4.5_uncal: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - shear: - path: v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - ra_col: RA - dec_col: Dec - e1_col: e1_uncal - e2_col: e2_uncal - w_col: w_des - R: 1.0 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - patch_number: 100 - -SP_v1.4.7_uncal: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - shear: - path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - ra_col: RA - dec_col: Dec - e1_col: e1_uncal - e2_col: e2_uncal - w_col: w_des - R: 1.0 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - patch_number: 100 - -SP_v1.4.8_uncal: - pipeline: SP - subdir: /n17data/UNIONS/WL/v1.4.x - shear: - path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits - covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt - ra_col: RA - dec_col: Dec - e1_col: e1_uncal - e2_col: e2_uncal - w_col: w_des - R: 1.0 - psf: - path: unions_shapepipe_psf_2024_v1.4.a.fits - hdu: 1 - patch_number: 100 - -======= SP_v1.4.11.2: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -1105,7 +1049,6 @@ SP_v1.4.6.3_uncal_w_1: path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 patch_number: 100 ->>>>>>> Stashed changes SP_v1.4_LFmask_8k: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask pipeline: SP From 0ec3e3f21577708c474e5ff69450db9549dfe6f2 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 20 Apr 2026 11:30:53 +0200 Subject: [PATCH 03/14] added fill_photoz script --- scripts/fill_photoz_bands.py | 509 +++++++++++++++++++++++++++++++++++ 1 file changed, 509 insertions(+) create mode 100755 scripts/fill_photoz_bands.py diff --git a/scripts/fill_photoz_bands.py b/scripts/fill_photoz_bands.py new file mode 100755 index 00000000..83865b15 --- /dev/null +++ b/scripts/fill_photoz_bands.py @@ -0,0 +1,509 @@ +#!/usr/bin/env python + +"""fill_photoz_bands.py + +Add PhotoPipe photo-z and multi-band magnitude fields to a ShapePipe +comprehensive HDF5 catalogue and fill them from PhotoPipe FITS tiles. + +Two phases, both resumable via a checkpoint file: + + Phase 1 — Create output: copy the input HDF5 and append the new + columns initialised to EMPTY_VALUE (-199). + Phase 2 — Fill tiles: for each PhotoPipe FITS tile found in fits_dir, + write the field values into the output file. + +Skips missing FITS files. Warns (does not error) on missing PhotoPipe +keys. Supports interrupt + restart at any point. + +:Authors: Martin Kilbinger + +""" + +import json +import os +import sys +import warnings +from collections import defaultdict +from timeit import default_timer as timer + +import h5py +import numpy as np +import tqdm +from astropy.io import fits + +from cs_util import args as cs_args +from cs_util import logging + + +FITS_HDU = 1 +EMPTY_VALUE = -199 +COPY_CHUNK = 2_000_000 # rows per chunk when copying input → output +SCAN_CHUNK = 5_000_000 # rows per chunk when scanning TILE_ID + +REQUESTED_KEYS = [ + "Z_B", + "Z_B_MIN", + "Z_B_MAX", + "T_B", + "MAG_GAAP_0p7_u", + "MAG_GAAP_0p7_g", + "MAG_GAAP_0p7_r", + "MAG_GAAP_0p7_i", + "MAG_GAAP_0p7_z", + "MAG_GAAP_0p7_z2", +] + + +def params_default(): + """Params Default. + + Return default parameter values and additional information + about type and command line options. + + Returns + ------- + tuple + parameter dict, short_options dict, types dict, help_strings dict + + """ + params = { + "input": "unions_shapepipe_comprehensive_struc_2024_v1.5.c.hdf5", + "output": "unions_shapepipe_comprehensive_struc_ugriz_2024_v1.5.c.hdf5", + "fits_dir": "UNIONS5000", + "checkpoint": "fill_photoz_bands_checkpoint.json", + "verbose": False, + } + + short_options = { + "input": "-i", + "output": "-o", + "fits_dir": "-d", + "checkpoint": "-c", + } + + types = {} + + help_strings = { + "input": "input HDF5 catalogue (no PhotoPipe fields), default={}", + "output": "output HDF5 catalogue (PhotoPipe fields added and filled), default={}", + "fits_dir": "directory with PhotoPipe FITS tiles, default={}", + "checkpoint": "checkpoint JSON file for resume support, default={}", + } + + return params, short_options, types, help_strings + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def detect_dataset_name(hf): + """Return the first dataset name in an HDF5 file. + + Tries common names first: dat, dat_comb, data. + + Parameters + ---------- + hf : h5py.File + + Returns + ------- + str + Dataset name. + + Raises + ------ + KeyError + If no dataset is found. + + """ + for name in ("dat", "dat_comb", "data"): + if name in hf: + return name + # Fall back to first key + keys = list(hf.keys()) + if not keys: + raise KeyError("HDF5 file contains no datasets.") + return keys[0] + + +def strip_dtype(dtype): + """Strip h5py metadata from a structured numpy dtype descriptor. + + h5py sometimes adds encoding metadata to dtype descriptions, e.g. + ``(' 1)[0] + 1 + blocks = np.split(sorted_idx, gaps) + fits_offset = 0 + for block in blocks: + b_min, b_max = int(block[0]), int(block[-1]) + n_block = b_max - b_min + 1 + chunk = dset[b_min : b_max + 1] + for key in valid_keys: + chunk[key] = fits_data[key][fits_offset : fits_offset + n_block] + dset[b_min : b_max + 1] = chunk + fits_offset += n_block + + +# --------------------------------------------------------------------------- +# Phase 1: create output file +# --------------------------------------------------------------------------- + +def create_output_file(input_path, output_path, dataset_name, verbose=False): + """Create output HDF5 by copying input and appending empty PhotoPipe fields. + + Parameters + ---------- + input_path : str + output_path : str + dataset_name : str + Dataset name in the input file (used for output too). + verbose : bool + + """ + print(f"Phase 1: creating output file '{output_path}'") + t0 = timer() + + with h5py.File(input_path, "r") as hf_in: + dset_in = hf_in[dataset_name] + n_total = dset_in.shape[0] + dtype_out = build_output_dtype(dset_in.dtype, REQUESTED_KEYS) + new_keys = [k for k in REQUESTED_KEYS if k not in set(dset_in.dtype.names)] + + print(f" Input rows : {n_total:,}") + print(f" Input fields : {len(dset_in.dtype.names)}") + print(f" New fields : {new_keys}") + size_mb = n_total * dtype_out.itemsize / 1_048_576 + print(f" Output size : ~{size_mb:,.0f} MB") + + with h5py.File(output_path, "w") as hf_out: + dset_out = hf_out.create_dataset( + dataset_name, + shape=(n_total,), + dtype=dtype_out, + ) + + # Copy input fields chunk by chunk + input_fields = dset_in.dtype.names + with tqdm.tqdm( + total=n_total, unit="rows", unit_scale=True, desc=" Copying" + ) as pbar: + for start in range(0, n_total, COPY_CHUNK): + end = min(start + COPY_CHUNK, n_total) + chunk_in = dset_in[start:end] + chunk_out = np.empty(end - start, dtype=dtype_out) + for field in input_fields: + chunk_out[field] = chunk_in[field] + for key in new_keys: + chunk_out[key] = EMPTY_VALUE + dset_out[start:end] = chunk_out + pbar.update(end - start) + + elapsed = timer() - t0 + print(f" Done in {elapsed:.1f}s\n") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(argv=None): + """Main. + + Main program. + + """ + params, short_options, types, help_strings = params_default() + + options = cs_args.parse_options(params, short_options, types, help_strings) + params.update(options) + + logging.log_command(argv) + + verbose = params["verbose"] + + if not os.path.exists(params["input"]): + print(f"ERROR: input file not found: {params['input']}", file=sys.stderr) + return 1 + if not os.path.isdir(params["fits_dir"]): + print(f"ERROR: FITS directory not found: {params['fits_dir']}", file=sys.stderr) + return 1 + + # ------------------------------------------------------------------ + # Load checkpoint + # ------------------------------------------------------------------ + if os.path.exists(params["checkpoint"]): + with open(params["checkpoint"]) as f: + checkpoint = json.load(f) + done_tiles = set(checkpoint.get("done_tiles", [])) + print(f"Resuming: {len(done_tiles)} tiles already completed.") + else: + done_tiles = set() + checkpoint = {} + + # ------------------------------------------------------------------ + # Detect input dataset name (cache in checkpoint) + # ------------------------------------------------------------------ + if "dataset_name" in checkpoint: + dataset_name = checkpoint["dataset_name"] + else: + with h5py.File(params["input"], "r") as hf: + dataset_name = detect_dataset_name(hf) + checkpoint["dataset_name"] = dataset_name + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + + if verbose: + print(f"Input dataset: '{dataset_name}'") + + # ------------------------------------------------------------------ + # Phase 1: create output file if needed + # ------------------------------------------------------------------ + if not checkpoint.get("output_created", False): + if os.path.exists(params["output"]): + print( + f"WARNING: output file '{params['output']}' exists but checkpoint " + "does not mark it as complete. Overwriting." + ) + create_output_file( + params["input"], params["output"], dataset_name, verbose=verbose + ) + checkpoint["output_created"] = True + # Invalidate any stale tile_index_map from a previous partial run + checkpoint.pop("tile_index_map", None) + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + else: + if verbose: + print("Phase 1 already done (output file exists in checkpoint).") + + # ------------------------------------------------------------------ + # Phase 2: fill tiles + # ------------------------------------------------------------------ + t0 = timer() + with h5py.File(params["output"], "r+") as hf: + dset = hf[dataset_name] + n_total = dset.shape[0] + print(f"Phase 2: filling tiles in '{params['output']}'") + print(f" {n_total:,} rows, dataset '{dataset_name}'") + + hdf5_fields = set(dset.dtype.names) + + # Check all requested keys are present + for key in REQUESTED_KEYS: + if key not in hdf5_fields: + warnings.warn( + f"Key '{key}' missing from output dataset — was Phase 1 complete?" + ) + + # Build TILE_ID → output row indices (scan once; cached in checkpoint) + if "tile_index_map" in checkpoint: + if verbose: + print(" Loading tile→index map from checkpoint...") + tile_index_map = { + k.encode(): np.array(v, dtype=np.int64) + for k, v in checkpoint["tile_index_map"].items() + } + else: + print(" Building tile→index map (scans all rows)...") + tile_index_map_lists = defaultdict(list) + with tqdm.tqdm( + total=n_total, unit="rows", unit_scale=True, desc=" Scanning TILE_ID" + ) as pbar: + for start in range(0, n_total, SCAN_CHUNK): + end = min(start + SCAN_CHUNK, n_total) + tile_chunk = dset["TILE_ID"][start:end] + for local_i, tid in enumerate(tile_chunk): + tile_index_map_lists[tid].append(start + local_i) + pbar.update(end - start) + + tile_index_map = { + tid: np.array(idxs, dtype=np.int64) + for tid, idxs in tile_index_map_lists.items() + } + checkpoint["tile_index_map"] = { + tid.decode(): idxs.tolist() + for tid, idxs in tile_index_map.items() + } + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + print(f" Map saved: {len(tile_index_map)} unique tiles.") + + unique_tiles = sorted(tile_index_map.keys()) + n_tiles = len(unique_tiles) + + valid_keys = None # determined from first available FITS tile + n_skipped_missing = 0 + n_skipped_done = 0 + n_skipped_size = 0 + n_processed = 0 + + print(f"\n Processing {n_tiles} tiles ({len(done_tiles)} already done)...\n") + pbar = tqdm.tqdm(unique_tiles, total=n_tiles, unit="tile") + + for tile_id in pbar: + tile_str = tile_id.decode() if isinstance(tile_id, bytes) else tile_id + + if tile_str in done_tiles: + n_skipped_done += 1 + continue + + fits_path = tile_id_to_fits_path(tile_id, params["fits_dir"]) + if not os.path.exists(fits_path): + n_skipped_missing += 1 + done_tiles.add(tile_str) + pbar.set_postfix({"done": n_processed, "missing": n_skipped_missing}) + continue + + try: + with fits.open(fits_path, memmap=True) as hdu_list: + fits_data = hdu_list[FITS_HDU].data + + # Validate keys on first successfully opened tile + if valid_keys is None: + valid_keys, missing_keys = check_fits_keys( + fits_data.dtype.names, REQUESTED_KEYS + ) + valid_keys = [k for k in valid_keys if k in hdf5_fields] + if missing_keys: + warnings.warn( + f"Keys absent from PhotoPipe FITS (skipped): {missing_keys}" + ) + tqdm.tqdm.write(f"\n Keys to fill: {valid_keys}\n") + + hdf5_indices = tile_index_map[tile_id] + n_hdf5 = len(hdf5_indices) + n_fits = len(fits_data) + + if n_hdf5 != n_fits: + warnings.warn( + f"Tile {tile_str}: HDF5 has {n_hdf5} rows, FITS has " + f"{n_fits} — size mismatch, skipping." + ) + n_skipped_size += 1 + pbar.set_postfix( + {"done": n_processed, "size_err": n_skipped_size} + ) + continue + + write_tile_to_hdf5(dset, hdf5_indices, fits_data, valid_keys) + + except Exception as e: + warnings.warn(f"Tile {tile_str}: error ({e}), skipping.") + continue + + n_processed += 1 + done_tiles.add(tile_str) + + if n_processed % 50 == 0: + checkpoint["done_tiles"] = list(done_tiles) + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + + pbar.set_postfix({"done": n_processed, "missing": n_skipped_missing}) + + # Final checkpoint flush + checkpoint["done_tiles"] = list(done_tiles) + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + + elapsed = timer() - t0 + print(f"\nDone in {elapsed:.1f}s") + print(f" Tiles processed : {n_processed}") + print(f" Tiles skipped (done) : {n_skipped_done}") + print(f" FITS files missing : {n_skipped_missing}") + print(f" Size mismatches : {n_skipped_size}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From 7f29a8b58dcbdfc00d601a8cd489ca3b99357225 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 16 Jun 2026 13:41:30 +0200 Subject: [PATCH 04/14] fill photoz bands fixes --- notebooks/create_shear_mb_empty.py | 172 --------------------------- notebooks/demo_add_bands_to_empty.py | 17 +-- scripts/fill_photoz_bands.py | 59 ++++----- 3 files changed, 34 insertions(+), 214 deletions(-) delete mode 100644 notebooks/create_shear_mb_empty.py diff --git a/notebooks/create_shear_mb_empty.py b/notebooks/create_shear_mb_empty.py deleted file mode 100644 index 6d29bfc2..00000000 --- a/notebooks/create_shear_mb_empty.py +++ /dev/null @@ -1,172 +0,0 @@ -# --- -# jupyter: -# jupytext: -# text_representation: -# extension: .py -# format_name: light -# format_version: '1.5' -# jupytext_version: 1.15.1 -# kernelspec: -# display_name: sp_validation -# language: python -# name: sp_validation -# --- - -# # Demo notebook to add (u,g,i,z,z2) bands to an r-band catalogue - -# %reload_ext autoreload -# %autoreload 2 - -# + -import os -import numpy as np -import numpy.lib.recfunctions as rfn -import h5py - -from timeit import default_timer as timer -import tqdm -import healsparse as hsp -from astropy.io import fits - -from sp_validation import run_joint_cat as sp_joint -# - - -# Create instance of object -obj = sp_joint.BaseCat() - - -# + -# Set parameters -base = "unions_shapepipe_comprehensive_struc" -year = 2024 -ver = "v1.6.c" - -obj._params = {} - -obj._params["input_path"] = f"{base}_{year}_{ver}.hdf5" -obj._params["verbose"] = True - -# + -path_bands = "./UNIONS5000" -subdir_base = "UNIONS." - -path_base = subdir_base -path_suff = "_SP_ugriz_photoz_ext.cat" - -# NUMBER key in photo-z catalogue -key_num = "SeqNr" - -keys_mag = [f"MAG_GAAP_0p7_{band}" for band in ("u", "g", "r", "i", "z", "z2")] - -keys = ["Z_B", "Z_B_MIN", "Z_B_MAX", "T_B"] + keys_mag - -hdu_no = 1 -# - - -# ## Run - -# + -# Check parameter validity -#obj.check_params() - -# Update parameters (here: strings to list) -#obj.update_params() -# - - -# Read catalogue -dat = obj.read_cat(load_into_memory=False, mode="r") -n_rows = len(dat) - - -def get_dtype_keys(keys,path=None, hdu_no=1): - - if path is None: - - dtype = np.dtype([(key, np.float32) for key in keys]) - - else: - - print(" Read data from file:", path, end=" ") - start = timer() - hdu_list = fits.open(path) - dat_mb = hdu_list[hdu_no].data - dtype = np.dtype([dt for dt in dat_mb.dtype.descr if dt[0] in keys]) - end = timer() - print(f" {end - start:.1f}s") - - return dtype - - -# + -# Get dtype of new keys - -path = None -#path = os.path.join(path_bands, f"{path_base}{tile_ID}", f"{path_base}{tile_IDs[0]}{path_suff}") - -dtype_keys = get_dtype_keys(keys, path=path, hdu_no=hdu_no) - - -# - - -def strip_h5py_metadata_dtype(dat_dtype, dat_ext_dtype): - cleaned_fields = [] - for name, dt in dat_dtype.descr + dat_ext_dtype.descr: - # If dt is a tuple (e.g., ('S7', {'h5py_encoding': 'ascii'})) - if isinstance(dt, tuple): - cleaned_fields.append((name, dt[0])) # keep only the base dtype string - else: - cleaned_fields.append((name, dt)) # use as-is - return cleaned_fields - - -# + -# Create empty array with new keys -# Initialise with -199 to later be able to check for unfilled values - -total_bytes = n_rows * np.dtype(dtype_keys).itemsize -print(" Create new combined array.", end=" ") -print(f"Expected size = {total_bytes / 1_048_576:.2f} MB", end=" ") -start = timer() - -obj._params["output_path"] = f"{base}_empty_ugriz_{year}_{ver}.hdf5" -dtype_sp = dat.dtype -dtype_comb = strip_h5py_metadata_dtype(dtype_sp, dtype_keys) -with h5py.File(obj._params["output_path"], "w") as f: - - # Create new dataset - dset_comb = f.create_dataset( - "dat_comb", - shape=(n_rows,), - dtype=dtype_comb, - ) - - # Copy old data field-by-field - for name in dtype_sp.names: - dset_comb[name] = dat[name] - - # Fill new fields with default value (-199) - for name in dtype_keys.names: - dset_comb[name] = -199 - -#new_empty = np.full(n_rows, -199, dtype=dtype_keys) - -end = timer() -print(f" {end - start:.1f}s") - - - -# + -# Merge with original data - -#print(" Merge empty to original", end=" ") -#start = timer() -#combined = rfn.merge_arrays([dat, new_empty], flatten=True) -#end = timer() -#print(f" {end - start:.1f}s") -# - - -# obj.write_hdf5_file(combined) - - -# Close input HDF5 catalogue file -# obj.close_hd5() diff --git a/notebooks/demo_add_bands_to_empty.py b/notebooks/demo_add_bands_to_empty.py index 14788826..3ea24ed6 100644 --- a/notebooks/demo_add_bands_to_empty.py +++ b/notebooks/demo_add_bands_to_empty.py @@ -43,7 +43,7 @@ # Set parameters base = "unions_shapepipe_comprehensive_struc_empty_ugriz" year = 2024 -ver = "v1.5.c" +ver = "v1.6.c" obj._params = {} @@ -59,19 +59,20 @@ bands = ("u", "g", "r", "i", "z", "z2") -base_keys = ["MAGERR_GAAP", "FLUX_GAAP", "FLUXERR_GAAP", "FLAG_GAAP", "MAG_LIM"] keys_mag = [f"MAG_GAAP_0p7_{band}" for band in bands] -for base_key in base_keys: - keys_mag.extend([f"_{base_key}_{band}" for band in bands]) +#base_keys = ["MAGERR_GAAP", "FLUX_GAAP", "FLUXERR_GAAP", "FLAG_GAAP", "MAG_LIM"] +#for base_key in base_keys: + #keys_mag.extend([f"{base_key}_{band}" for band in bands]) + +# "EXTINCTION", +# "MP_NAME", +# "ODDS", keys = [ - "EXTINCTION", - "MP_NAME", "Z_B", "Z_B_MIN", - "Z_B_MAX" + "Z_B_MAX", "T_B", - "ODDS", ] + keys_mag hdu_no = 1 diff --git a/scripts/fill_photoz_bands.py b/scripts/fill_photoz_bands.py index 83865b15..31ab290e 100755 --- a/scripts/fill_photoz_bands.py +++ b/scripts/fill_photoz_bands.py @@ -287,6 +287,13 @@ def create_output_file(input_path, output_path, dataset_name, verbose=False): dset_out[start:end] = chunk_out pbar.update(end - start) + # Copy all other top-level datasets/groups unchanged + for key in hf_in.keys(): + if key != dataset_name: + hf_in.copy(key, hf_out) + if verbose: + print(f" Copied group/dataset '{key}' unchanged.") + elapsed = timer() - t0 print(f" Done in {elapsed:.1f}s\n") @@ -357,8 +364,6 @@ def main(argv=None): params["input"], params["output"], dataset_name, verbose=verbose ) checkpoint["output_created"] = True - # Invalidate any stale tile_index_map from a previous partial run - checkpoint.pop("tile_index_map", None) with open(params["checkpoint"], "w") as cf: json.dump(checkpoint, cf) else: @@ -384,38 +389,24 @@ def main(argv=None): f"Key '{key}' missing from output dataset — was Phase 1 complete?" ) - # Build TILE_ID → output row indices (scan once; cached in checkpoint) - if "tile_index_map" in checkpoint: - if verbose: - print(" Loading tile→index map from checkpoint...") - tile_index_map = { - k.encode(): np.array(v, dtype=np.int64) - for k, v in checkpoint["tile_index_map"].items() - } - else: - print(" Building tile→index map (scans all rows)...") - tile_index_map_lists = defaultdict(list) - with tqdm.tqdm( - total=n_total, unit="rows", unit_scale=True, desc=" Scanning TILE_ID" - ) as pbar: - for start in range(0, n_total, SCAN_CHUNK): - end = min(start + SCAN_CHUNK, n_total) - tile_chunk = dset["TILE_ID"][start:end] - for local_i, tid in enumerate(tile_chunk): - tile_index_map_lists[tid].append(start + local_i) - pbar.update(end - start) - - tile_index_map = { - tid: np.array(idxs, dtype=np.int64) - for tid, idxs in tile_index_map_lists.items() - } - checkpoint["tile_index_map"] = { - tid.decode(): idxs.tolist() - for tid, idxs in tile_index_map.items() - } - with open(params["checkpoint"], "w") as cf: - json.dump(checkpoint, cf) - print(f" Map saved: {len(tile_index_map)} unique tiles.") + # Build TILE_ID → output row indices (always scanned; too large for checkpoint) + print(" Building tile→index map (scans all rows)...") + tile_index_map_lists = defaultdict(list) + with tqdm.tqdm( + total=n_total, unit="rows", unit_scale=True, desc=" Scanning TILE_ID" + ) as pbar: + for start in range(0, n_total, SCAN_CHUNK): + end = min(start + SCAN_CHUNK, n_total) + tile_chunk = dset[start:end]["TILE_ID"] + for local_i, tid in enumerate(tile_chunk): + tile_index_map_lists[tid].append(start + local_i) + pbar.update(end - start) + + tile_index_map = { + tid: np.array(idxs, dtype=np.int64) + for tid, idxs in tile_index_map_lists.items() + } + print(f" Map built: {len(tile_index_map)} unique tiles.") unique_tiles = sorted(tile_index_map.keys()) n_tiles = len(unique_tiles) From 7e88b056432f1a4a88d1785cd2ffe8e3bb3e8317 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Thu, 25 Jun 2026 15:30:28 +0200 Subject: [PATCH 05/14] adding mag errors to fill_photoz --- scripts/fill_photoz_bands.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/fill_photoz_bands.py b/scripts/fill_photoz_bands.py index 31ab290e..627ac3cf 100755 --- a/scripts/fill_photoz_bands.py +++ b/scripts/fill_photoz_bands.py @@ -45,12 +45,18 @@ "Z_B_MIN", "Z_B_MAX", "T_B", - "MAG_GAAP_0p7_u", - "MAG_GAAP_0p7_g", - "MAG_GAAP_0p7_r", - "MAG_GAAP_0p7_i", - "MAG_GAAP_0p7_z", - "MAG_GAAP_0p7_z2", + "MAG_GAAP_u", + "MAGERR_GAAP_u", + "MAG_GAAP_g", + "MAGERR_GAAP_g", + "MAG_GAAP_r", + "MAGERR_GAAP_r", + "MAG_GAAP_i", + "MAGERR_GAAP_i", + "MAG_GAAP_z", + "MAGERR_GAAP_z", + "MAG_GAAP_z2", + "MAGERR_GAAP_z2", ] From 9798047c676ce0cadfd48774fc301c5fe141c34d Mon Sep 17 00:00:00 2001 From: "martin.kilbinger@cea.fr" Date: Fri, 26 Jun 2026 10:53:20 +0200 Subject: [PATCH 06/14] added more flags to fill_photoz --- scripts/fill_photoz_bands.py | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/scripts/fill_photoz_bands.py b/scripts/fill_photoz_bands.py index 627ac3cf..d11af42d 100755 --- a/scripts/fill_photoz_bands.py +++ b/scripts/fill_photoz_bands.py @@ -47,16 +47,54 @@ "T_B", "MAG_GAAP_u", "MAGERR_GAAP_u", + "FLAG_GAAP_u", + "MAG_LIM_u", + "FLUX_GAAP_u", + "FLUXERR_GAAP_u", + "EXTINCTION_u", "MAG_GAAP_g", "MAGERR_GAAP_g", + "FLAG_GAAP_g", + "MAG_LIM_g", + "FLUX_GAAP_g", + "FLUXERR_GAAP_g", + "EXTINCTION_g", "MAG_GAAP_r", "MAGERR_GAAP_r", + "FLAG_GAAP_r", + "MAG_LIM_r", + "FLUX_GAAP_r", + "FLUXERR_GAAP_r", + "EXTINCTION_r", "MAG_GAAP_i", "MAGERR_GAAP_i", + "FLAG_GAAP_i", + "MAG_LIM_i", + "FLUX_GAAP_i", + "FLUXERR_GAAP_i", + "EXTINCTION_i", "MAG_GAAP_z", "MAGERR_GAAP_z", + "FLAG_GAAP_z", + "MAG_LIM_z", + "FLUX_GAAP_z", + "FLUXERR_GAAP_z", + "EXTINCTION_z", "MAG_GAAP_z2", "MAGERR_GAAP_z2", + "FLAG_GAAP_z2", + "MAG_LIM_z2", + "FLUX_GAAP_z2", + "FLUXERR_GAAP_z2", + "EXTINCTION_z2", + "EXTINCTION", + "ODDS", + "CHI_SQUARED_BPZ", + "M_0", + "BPZ_FILT", + "BPZ_NONDETFILT", + "BPZ_FLAGFILT", + "MP_NAME", ] From 7cdcfdfa27555bb5d51b5c36c66b2165f22fd9a0 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 29 Jun 2026 08:51:13 +0200 Subject: [PATCH 07/14] added 0p7 and 1p0 aperture magnitudes to PhotoPipe + SP output --- scripts/fill_photoz_bands.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/scripts/fill_photoz_bands.py b/scripts/fill_photoz_bands.py index d11af42d..761b0308 100755 --- a/scripts/fill_photoz_bands.py +++ b/scripts/fill_photoz_bands.py @@ -45,48 +45,79 @@ "Z_B_MIN", "Z_B_MAX", "T_B", + "MAG_GAAP_u", "MAGERR_GAAP_u", + "MAG_GAAP_0p7_u", + "MAGERR_GAAP_0p7_u", + "MAG_GAAP_1p0_u", + "MAGERR_GAAP_1p0_u", "FLAG_GAAP_u", "MAG_LIM_u", "FLUX_GAAP_u", "FLUXERR_GAAP_u", "EXTINCTION_u", + "MAG_GAAP_g", "MAGERR_GAAP_g", + "MAG_GAAP_0p7_g", + "MAGERR_GAAP_0p7_g", + "MAG_GAAP_1p0_g", + "MAGERR_GAAP_1p0_g", "FLAG_GAAP_g", "MAG_LIM_g", "FLUX_GAAP_g", "FLUXERR_GAAP_g", "EXTINCTION_g", + "MAG_GAAP_r", "MAGERR_GAAP_r", + "MAG_GAAP_0p7_r", + "MAGERR_GAAP_0p7_r", + "MAG_GAAP_1p0_r", + "MAGERR_GAAP_1p0_r", "FLAG_GAAP_r", "MAG_LIM_r", "FLUX_GAAP_r", "FLUXERR_GAAP_r", "EXTINCTION_r", + "MAG_GAAP_i", "MAGERR_GAAP_i", + "MAG_GAAP_0p7_i", + "MAGERR_GAAP_0p7_i", + "MAG_GAAP_1p0_i", + "MAGERR_GAAP_1p0_i", "FLAG_GAAP_i", "MAG_LIM_i", "FLUX_GAAP_i", "FLUXERR_GAAP_i", "EXTINCTION_i", + "MAG_GAAP_z", "MAGERR_GAAP_z", + "MAG_GAAP_0p7_z", + "MAGERR_GAAP_0p7_z", + "MAG_GAAP_1p0_z", + "MAGERR_GAAP_1p0_z", "FLAG_GAAP_z", "MAG_LIM_z", "FLUX_GAAP_z", "FLUXERR_GAAP_z", "EXTINCTION_z", + "MAG_GAAP_z2", "MAGERR_GAAP_z2", + "MAG_GAAP_0p7_z2", + "MAGERR_GAAP_0p7_z2", + "MAG_GAAP_1p0_z2", + "MAGERR_GAAP_1p0_z2", "FLAG_GAAP_z2", "MAG_LIM_z2", "FLUX_GAAP_z2", "FLUXERR_GAAP_z2", "EXTINCTION_z2", + "EXTINCTION", "ODDS", "CHI_SQUARED_BPZ", From bfae66522c6d62ebe937b02ee3fc68c554c60627 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 6 Jul 2026 14:41:06 +0200 Subject: [PATCH 08/14] Fixed fill photoz script with new MP_NAME type --- notebooks/calibrate_comprehensive_cat.py | 4 +- scripts/check_filled_fields.py | 72 ++++++++++++++++++++++++ scripts/fill_photoz_bands.py | 34 ++++++++--- 3 files changed, 99 insertions(+), 11 deletions(-) create mode 100644 scripts/check_filled_fields.py diff --git a/notebooks/calibrate_comprehensive_cat.py b/notebooks/calibrate_comprehensive_cat.py index 404a7bec..5a0de543 100644 --- a/notebooks/calibrate_comprehensive_cat.py +++ b/notebooks/calibrate_comprehensive_cat.py @@ -47,8 +47,8 @@ dat, dat_ext = obj.read_cat(load_into_memory=False) # %% -n_test = -1 -#n_test = 100000 +#n_test = -1 +n_test = 100000 if n_test > 0: print(f"MKDEBUG testing only first {n_test} objects") dat = dat[:n_test] diff --git a/scripts/check_filled_fields.py b/scripts/check_filled_fields.py new file mode 100644 index 00000000..c998c26f --- /dev/null +++ b/scripts/check_filled_fields.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Check how many entries in selected HDF5 fields are filled (value != -199). + +Reads in chunks with a progress bar and periodic fill-fraction updates. +""" + +import h5py +import numpy as np +from tqdm import tqdm + +HDF5_FILE = "unions_shapepipe_comprehensive_struc_ugriz_2024_v1.6.c.DR6.hdf5" +EMPTY_VALUE = -199 +CHUNK_SIZE = 5_000_000 # rows per chunk +REPORT_EVERY = 10 # print running fractions every N chunks + +FIELDS = [ + "Z_B", + "Z_B_MIN", + "Z_B_MAX", + "T_B", + "MAG_GAAP_0p7_u", + "MAG_GAAP_1p0_u", + "MAG_GAAP_0p7_g", + "MAG_GAAP_1p0_g", + "MAG_GAAP_0p7_r", + "MAG_GAAP_1p0_r", + "MAG_GAAP_0p7_i", + "MAG_GAAP_1p0_i", + "MAG_GAAP_0p7_z", + "MAG_GAAP_1p0_z", + "MAG_GAAP_0p7_z2", + "MAG_GAAP_1p0_z2", +] + +with h5py.File(HDF5_FILE, "r") as f: + data = f["data"] + n_total = data.shape[0] + n_chunks = (n_total + CHUNK_SIZE - 1) // CHUNK_SIZE + print(f"Total entries : {n_total:,}") + print(f"Chunk size : {CHUNK_SIZE:,} ({n_chunks} chunks)\n") + + counts = {field: 0 for field in FIELDS} + + with tqdm(total=n_total, unit="rows", unit_scale=True, desc="Reading") as pbar: + for chunk_idx in range(n_chunks): + start = chunk_idx * CHUNK_SIZE + end = min(start + CHUNK_SIZE, n_total) + + for field in FIELDS: + counts[field] += int(np.sum(data[field, start:end] != EMPTY_VALUE)) + + pbar.update(end - start) + + # Periodic running-fraction report + if (chunk_idx + 1) % REPORT_EVERY == 0 or (chunk_idx + 1) == n_chunks: + rows_done = end + tqdm.write(f"\n --- after {rows_done:,} rows ({100*rows_done/n_total:.1f}%) ---") + tqdm.write(f" {'Field':<22} {'Filled %':>9}") + for field in FIELDS: + pct = 100.0 * counts[field] / rows_done + tqdm.write(f" {field:<22} {pct:>8.2f}%") + +# Final summary +print(f"\n{'='*56}") +print(f"FINAL SUMMARY (total rows: {n_total:,})") +print(f"{'Field':<22} {'Filled':>12} {'Empty':>12} {'Filled %':>10}") +print("-" * 60) +for field in FIELDS: + n_filled = counts[field] + n_empty = n_total - n_filled + pct = 100.0 * n_filled / n_total + print(f"{field:<22} {n_filled:>12,} {n_empty:>12,} {pct:>9.2f}%") diff --git a/scripts/fill_photoz_bands.py b/scripts/fill_photoz_bands.py index 761b0308..f29c9e4a 100755 --- a/scripts/fill_photoz_bands.py +++ b/scripts/fill_photoz_bands.py @@ -39,13 +39,13 @@ EMPTY_VALUE = -199 COPY_CHUNK = 2_000_000 # rows per chunk when copying input → output SCAN_CHUNK = 5_000_000 # rows per chunk when scanning TILE_ID +MAX_CONSEC_FAILS = 10 # abort if this many tiles in a row fail REQUESTED_KEYS = [ "Z_B", "Z_B_MIN", "Z_B_MAX", "T_B", - "MAG_GAAP_u", "MAGERR_GAAP_u", "MAG_GAAP_0p7_u", @@ -57,7 +57,6 @@ "FLUX_GAAP_u", "FLUXERR_GAAP_u", "EXTINCTION_u", - "MAG_GAAP_g", "MAGERR_GAAP_g", "MAG_GAAP_0p7_g", @@ -69,7 +68,6 @@ "FLUX_GAAP_g", "FLUXERR_GAAP_g", "EXTINCTION_g", - "MAG_GAAP_r", "MAGERR_GAAP_r", "MAG_GAAP_0p7_r", @@ -81,7 +79,6 @@ "FLUX_GAAP_r", "FLUXERR_GAAP_r", "EXTINCTION_r", - "MAG_GAAP_i", "MAGERR_GAAP_i", "MAG_GAAP_0p7_i", @@ -93,7 +90,6 @@ "FLUX_GAAP_i", "FLUXERR_GAAP_i", "EXTINCTION_i", - "MAG_GAAP_z", "MAGERR_GAAP_z", "MAG_GAAP_0p7_z", @@ -105,7 +101,6 @@ "FLUX_GAAP_z", "FLUXERR_GAAP_z", "EXTINCTION_z", - "MAG_GAAP_z2", "MAGERR_GAAP_z2", "MAG_GAAP_0p7_z2", @@ -117,7 +112,6 @@ "FLUX_GAAP_z2", "FLUXERR_GAAP_z2", "EXTINCTION_z2", - "EXTINCTION", "ODDS", "CHI_SQUARED_BPZ", @@ -125,7 +119,6 @@ "BPZ_FILT", "BPZ_NONDETFILT", "BPZ_FLAGFILT", - "MP_NAME", ] @@ -144,7 +137,7 @@ def params_default(): params = { "input": "unions_shapepipe_comprehensive_struc_2024_v1.5.c.hdf5", "output": "unions_shapepipe_comprehensive_struc_ugriz_2024_v1.5.c.hdf5", - "fits_dir": "UNIONS5000", + "fits_dir": "UNIONS_DR6", "checkpoint": "fill_photoz_bands_checkpoint.json", "verbose": False, } @@ -490,12 +483,25 @@ def main(argv=None): n_skipped_missing = 0 n_skipped_done = 0 n_skipped_size = 0 + n_errors = 0 + n_consec_fails = 0 n_processed = 0 print(f"\n Processing {n_tiles} tiles ({len(done_tiles)} already done)...\n") pbar = tqdm.tqdm(unique_tiles, total=n_tiles, unit="tile") for tile_id in pbar: + if n_consec_fails >= MAX_CONSEC_FAILS: + checkpoint["done_tiles"] = list(done_tiles) + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + print( + f"\nERROR: {n_consec_fails} tiles failed in a row, " + "likely a systematic problem; aborting.", + file=sys.stderr, + ) + sys.exit(1) + tile_str = tile_id.decode() if isinstance(tile_id, bytes) else tile_id if tile_str in done_tiles: @@ -535,6 +541,7 @@ def main(argv=None): f"{n_fits} — size mismatch, skipping." ) n_skipped_size += 1 + n_consec_fails += 1 pbar.set_postfix( {"done": n_processed, "size_err": n_skipped_size} ) @@ -544,9 +551,12 @@ def main(argv=None): except Exception as e: warnings.warn(f"Tile {tile_str}: error ({e}), skipping.") + n_errors += 1 + n_consec_fails += 1 continue n_processed += 1 + n_consec_fails = 0 done_tiles.add(tile_str) if n_processed % 50 == 0: @@ -567,6 +577,12 @@ def main(argv=None): print(f" Tiles skipped (done) : {n_skipped_done}") print(f" FITS files missing : {n_skipped_missing}") print(f" Size mismatches : {n_skipped_size}") + print(f" Tiles failed (error) : {n_errors}") + if n_processed == 0 and (n_errors > 0 or n_skipped_size > 0): + print( + "WARNING: no tiles were filled; all available tiles failed.", + file=sys.stderr, + ) return 0 From 82b7b156b10c7ecf54097c6f99c48602f0bc56d0 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Wed, 9 Sep 2026 18:47:41 +0200 Subject: [PATCH 09/14] comprehensive cat backwards compatible, maybe not use for merge --- config/calibration/mask_v1.X.6_ppv1.yaml | 6 ++++-- scripts/calibration/calibrate_comprehensive_cat.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/config/calibration/mask_v1.X.6_ppv1.yaml b/config/calibration/mask_v1.X.6_ppv1.yaml index a3d29cda..0708ee88 100644 --- a/config/calibration/mask_v1.X.6_ppv1.yaml +++ b/config/calibration/mask_v1.X.6_ppv1.yaml @@ -1,9 +1,11 @@ # Config file for masking and calibration. -# Standard cuts without halo masks, type v1.X.6. +# Joint ShapePipe + PhotoPipe catalogue. +# SP: Standard cuts without halo masks, type v1.X.6. +# PP" Standard selection. # General parameters (can also given on command line) params: - input_path: unions_shapepipe_comprehensive_struc_2024_v1.X.c.hdf5 + input_path: unions_shapepipe_comprehensive_struc_ugriz_2024_v1.X.c.hdf5 cmatrices: False sky_regions: False verbose: True diff --git a/scripts/calibration/calibrate_comprehensive_cat.py b/scripts/calibration/calibrate_comprehensive_cat.py index 2605338f..b9d44e2d 100644 --- a/scripts/calibration/calibrate_comprehensive_cat.py +++ b/scripts/calibration/calibrate_comprehensive_cat.py @@ -189,7 +189,7 @@ "FLUX_APER", "FLUXERR_APER", "NGMIX_T_NOSHEAR", - "NGMIX_T_PSF_RECONV_NOSHEAR", + "NGMIX_Tpsf_NOSHEAR", "fwhm_PSF", ] add_cols_data = {} From 055a1d524ad62b88e12657c91bd8f156181ca289 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sun, 20 Sep 2026 08:18:53 +0200 Subject: [PATCH 10/14] Using all photometric bands and masks --- config/calibration/mask_v1.X.6_ppv1.yaml | 32 ++++++++++ scripts/calibration/demo_apply_hsp_masks.py | 47 ++++++++------ scripts/fill_photoz_bands.py | 4 +- src/sp_validation/calibration.py | 49 +++++++++++++-- src/sp_validation/masks.py | 70 ++++++++++++++++++--- src/sp_validation/tests/test_masks.py | 49 +++++++++++++++ 6 files changed, 218 insertions(+), 33 deletions(-) diff --git a/config/calibration/mask_v1.X.6_ppv1.yaml b/config/calibration/mask_v1.X.6_ppv1.yaml index 0708ee88..0a7dc471 100644 --- a/config/calibration/mask_v1.X.6_ppv1.yaml +++ b/config/calibration/mask_v1.X.6_ppv1.yaml @@ -101,12 +101,44 @@ dat_ext: kind: equal value: False + # u-band footprint + - col_name: 16_u + label: "u-band imaging" + kind: equal + value: False + + # g-band footprint + - col_name: 32_g + label: "g-band imaging" + kind: equal + value: False + # r-band footprint - col_name: 64_r label: "r-band imaging" kind: equal value: False + # i-band footprint + - col_name: 128_i + label: "i-band imaging" + kind: equal + value: False + + # z-band footprint, both z surveys: 256_z is WISHES (Subaru HSC), 2048_z2 is + # Pan-STARRS (the deeper data, not the shallow all-sky survey). Told apart by + # depth: median MAG_LIM_z = 25.54 vs MAG_LIM_z2 = 24.88. + # Each bit is only required where its own band has a magnitude: asking for + # both at once keeps only the overlap of the two surveys, which costs ~85% of + # the sample. Pairs with the MAG_GAAP_0p7_z / _z2 two-band cut in `dat`. + - col_name: 256_z + col_name2: 2048_z2 + mag_col: MAG_GAAP_0p7_z + mag_col2: MAG_GAAP_0p7_z2 + label: "z-band imaging" + kind: covered_2bands + value: -99 + # Maximask - col_name: 1024_Maximask label: "maximask" diff --git a/scripts/calibration/demo_apply_hsp_masks.py b/scripts/calibration/demo_apply_hsp_masks.py index beafb909..3486fcf6 100644 --- a/scripts/calibration/demo_apply_hsp_masks.py +++ b/scripts/calibration/demo_apply_hsp_masks.py @@ -22,7 +22,6 @@ # + import os -import re import tracemalloc from sp_validation import catalog_builders as sp_joint @@ -39,41 +38,49 @@ obj = sp_joint.ApplyHspMasks() -# Bits set for non-tomographic catalogues -bit_list = [1, 2, 4, 8, 64, 1024] # + +# Set parameters +base = "unions_shapepipe_comprehensive" + +# Major version +ver_maj = "v1.6" + +# Mask release, subdirectory of the mask directory +mask_release = "dr6-2026-08" + +year = 2022 if ver_maj == "v1.3" else 2024 + +# $HOME/masks is a link to the shared UNIONS mask directory +mask_base = f"{os.environ['HOME']}/masks" + +if ver_maj in ("v1.5", "v1.6"): + # Multi-band (tomographic) + bit_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 1024, 2048] + obj._params["mask_dir"] = f"{mask_base}/{mask_release}" + obj._params["file_base"] = "mask_ugriz_" +else: + # r-band (non-tomographic) + bit_list = [1, 2, 4, 8, 64, 1024] + obj._params["mask_dir"] = mask_base + obj._params["file_base"] = "mask_r_" + # combine bit list with & operator # equivalent to bits = sum(bit_list) # if elements in bit_list are of type 2^n - bits = 0 for b in bit_list: bits = bits | b print(bits) -# + -# Set parameters -base = "unions_shapepipe_comprehensive" -ver = "v1.3.c" - -if ver == "v1.3.c": - year = 2022 -else: - year = 2024 -ver_maj = "v1.5" - obj._params["input_path"] = f"{base}_{year}_{ver_maj}.c.hdf5" obj._params["output_path"] = f"{base}_struc_{year}_{ver_maj}.c.hdf5" -obj._params["mask_dir"] = f"{os.environ['HOME']}/{ver_maj}.x/masks" obj._params["nside"] = 131072 -obj._params["file_base"] = "mask_r_" obj._params["bits"] = bits -obj._params["aux_mask_files"] = f"{obj._params['mask_dir']}/coverage_{ver_maj}.x.hsp" -ver_x = re.sub(r"c$", "x", ver) -obj._params["aux_mask_files"] = f"{obj._params['mask_dir']}/coverage_{ver_x}.hsp" +# Coverage map is in the mask base directory, not in the mask release one +obj._params["aux_mask_files"] = f"{mask_base}/coverage_{ver_maj}.x.hsp" obj._params["aux_mask_labels"] = "npoint3" obj._params["verbose"] = True # - diff --git a/scripts/fill_photoz_bands.py b/scripts/fill_photoz_bands.py index f29c9e4a..4ce1a66e 100755 --- a/scripts/fill_photoz_bands.py +++ b/scripts/fill_photoz_bands.py @@ -465,7 +465,9 @@ def main(argv=None): ) as pbar: for start in range(0, n_total, SCAN_CHUNK): end = min(start + SCAN_CHUNK, n_total) - tile_chunk = dset[start:end]["TILE_ID"] + # .fields() reads only TILE_ID; dset[start:end]["TILE_ID"] + # would materialise all columns first (~3 GB per chunk) + tile_chunk = dset.fields("TILE_ID")[start:end] for local_i, tid in enumerate(tile_chunk): tile_index_map_lists[tid].append(start + local_i) pbar.update(end - start) diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index e0ab212e..6147aa0e 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -687,6 +687,27 @@ def get_calibrate_no_leakage_e_from_cat(path_cat_gal, weight_type="des", verbose return e1_noleak, e2_noleak +def first_present(data, *candidates): + """Return the first of `candidates` present in `data`. + + ngmix columns come in two grammars: v1.6 and earlier catalogues use + NGMIX_ELL__ and NGMIX_Tpsf_, ShapePipe v2 writes + NGMIX_G_ and NGMIX_T_PSF_RECONV_. + """ + names = getattr(getattr(data, "dtype", None), "names", None) + if names is None: + names = getattr(data, "colnames", None) + if names is None: + names = list(data.keys()) + for candidate in candidates: + if candidate in names: + return candidate + raise KeyError( + f"none of {list(candidates)} in the catalogue; it has neither ngmix" + f" column grammar" + ) + + class metacal: """Metacal. @@ -816,10 +837,14 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): dict_tmp["flag"] = masked_data[f"{self._prefix}_FLAGS_{name_shear}"] - # Ellipticity in named scalar components (ShapePipe-v2 grammar) + # Either ngmix grammar (see first_present) for comp in (0, 1): dict_tmp[f"g{comp + 1}"] = masked_data[ - f"{self._prefix}_G{comp + 1}_{name_shear}" + first_present( + masked_data, + f"{self._prefix}_G{comp + 1}_{name_shear}", + f"{self._prefix}_ELL_{name_shear}_{comp}", + ) ] for key in ("flux", "flux_err", "T", "T_err"): @@ -827,7 +852,13 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): f"{self._prefix}_{key.upper()}_{name_shear}" ] - dict_tmp["Tpsf"] = masked_data[f"{self._prefix}_T_PSF_RECONV_{name_shear}"] + dict_tmp["Tpsf"] = masked_data[ + first_present( + masked_data, + f"{self._prefix}_T_PSF_RECONV_{name_shear}", + f"{self._prefix}_Tpsf_{name_shear}", + ) + ] ns["C11"], ns["C22"], ns["w"] = self.get_variance_ivweights( masked_data, @@ -875,8 +906,16 @@ def get_variance_ivweights(data, sigma_eps, prefix="NGMIX", mask=None): weight """ - C11 = data[f"{prefix}_G1_ERR_NOSHEAR"] - C22 = data[f"{prefix}_G2_ERR_NOSHEAR"] + C11 = data[ + first_present( + data, f"{prefix}_G1_ERR_NOSHEAR", f"{prefix}_ELL_ERR_NOSHEAR_0" + ) + ] + C22 = data[ + first_present( + data, f"{prefix}_G2_ERR_NOSHEAR", f"{prefix}_ELL_ERR_NOSHEAR_1" + ) + ] if mask is not None: C11 = C11[mask] C22 = C22[mask] diff --git a/src/sp_validation/masks.py b/src/sp_validation/masks.py index ddd09ef0..fcf99609 100644 --- a/src/sp_validation/masks.py +++ b/src/sp_validation/masks.py @@ -120,14 +120,22 @@ class Mask: kind : str operation type; see :func:`apply_condition` for the allowed values, plus "not_equal_2bands", which keeps objects whose value differs - from ``value`` in ``col_name`` *or* ``col_name2`` (two-band OR) + from ``value`` in ``col_name`` *or* ``col_name2`` (two-band OR), and + "covered_2bands", which keeps objects that have one band both + measured and inside that band's footprint (see :meth:`apply`) value : float or list value(s) to be used in mask operation col_name2 : str, optional - name of second column; required for (and only used by) kind - "not_equal_2bands" + name of second column; required for kinds "not_equal_2bands" and + "covered_2bands" + mag_col, mag_col2 : str, optional + magnitude column belonging to ``col_name`` and ``col_name2``; + required for (and only used by) kind "covered_2bands" dat : numpy.ndarray, optional input data, default is `None`; apply mask if given + dat_other : numpy.ndarray, optional + the catalogue's other group, holding ``mag_col``/``mag_col2`` when + those sit in a different group than the footprint bits verbose : bool, optional verbose output if ``True``; default is ``False`` @@ -140,7 +148,10 @@ def __init__( kind=None, value=0, col_name2=None, + mag_col=None, + mag_col2=None, dat=None, + dat_other=None, verbose=False, ): @@ -150,7 +161,20 @@ def __init__( self._kind = kind if kind == "not_equal_2bands" and col_name2 is None: raise ValueError("kind 'not_equal_2bands' requires col_name2") + if kind == "covered_2bands": + if None in (col_name2, mag_col, mag_col2): + raise ValueError( + "kind 'covered_2bands' requires col_name2, mag_col and mag_col2" + ) + if dat_other is None: + raise ValueError( + "kind 'covered_2bands' requires dat_other: the magnitudes sit " + "in a different catalogue group than the footprint bits" + ) self._col_name2 = col_name2 + self._mag_col = mag_col + self._mag_col2 = mag_col2 + self._dat_other = dat_other self._num_ok = None self._verbose = verbose @@ -179,12 +203,29 @@ def from_list(cls, masks, label="combined", verbose=False): return my_mask + def _band_measured_and_covered(self, dat, bit_col, mag_col): + """One band: magnitude present, and the footprint bit says covered. + + The bit is set where the band has no imaging, so a covered position + has it ``False``. + """ + measured = apply_condition(self._dat_other[mag_col], "not_equal", self._value) + return measured & ~np.asarray(dat[bit_col], dtype=bool) + def apply(self, dat): if self._kind == "not_equal_2bands": self._mask = apply_condition( dat[self._col_name], "not_equal", self._value ) | apply_condition(dat[self._col_name2], "not_equal", self._value) + elif self._kind == "covered_2bands": + # Each footprint bit is only asked about where its own band + # supplies a magnitude, so requiring both bands at once -- which + # no object outside the overlap of the two surveys can satisfy -- + # is avoided. Keep an object with at least one usable band. + self._mask = self._band_measured_and_covered( + dat, self._col_name, self._mag_col + ) | self._band_measured_and_covered(dat, self._col_name2, self._mag_col2) else: self._mask = apply_condition(dat[self._col_name], self._kind, self._value) @@ -254,6 +295,14 @@ def print_condition(self, f_out, latex=False): file=f_out, ) + if self._kind == "covered_2bands": + print( + f"({self._mag_col} != {self._value} and {name} = False)" + f" or ({self._mag_col2} != {self._value}" + f" and {self._col_name2} = False)", + file=f_out, + ) + if self._kind == "range": print(f"{self._value[0]} {sign} {name} {sign} {self._value[1]}", file=f_out) @@ -273,12 +322,16 @@ def create_descr(self): """ sign = self.get_sign() - if sign is not None: - descr = f"{sign}{self._value}" + descr = f"{sign}{self._value}" if sign is not None else str(self._kind) if self._kind == "range": descr = f"{self._value[0]}<={self._col_name}<={self._value[1]}" if self._kind == "not_equal_2bands": descr = f"!={self._value} in {self._col_name} or {self._col_name2}" + if self._kind == "covered_2bands": + descr = ( + f"{self._mag_col}!={self._value} and {self._col_name}=False, or " + f"{self._mag_col2}!={self._value} and {self._col_name2}=False" + ) self._descr = descr # Create description for FITS header @@ -347,8 +400,10 @@ def get_masks_from_config(config, dat, dat_ext, masks_to_apply=None, verbose=Fal config_data = {key: config[key] for key in ["dat", "dat_ext"] if key in config} idx = 0 for section, mask_list in config_data.items(): - # Set data source + # Set data source; the other group goes along for kinds that read a + # column from both (covered_2bands: bits here, magnitudes there) dat_source = dat if section == "dat" else dat_ext + dat_other = dat_ext if section == "dat" else dat # Loop over mask information in this section for mask_params in mask_list: @@ -370,7 +425,8 @@ def get_masks_from_config(config, dat, dat_ext, masks_to_apply=None, verbose=Fal ) # Create mask instance and append to list - my_mask = Mask(**mask_params, dat=dat_source, verbose=verbose) + my_mask = Mask(**mask_params, dat=dat_source, + dat_other=dat_other, verbose=verbose) masks.append(my_mask) labels[my_mask._col_name] = idx idx += 1 diff --git a/src/sp_validation/tests/test_masks.py b/src/sp_validation/tests/test_masks.py index 76e97748..edf8517e 100644 --- a/src/sp_validation/tests/test_masks.py +++ b/src/sp_validation/tests/test_masks.py @@ -97,3 +97,52 @@ def test_not_equal_2bands_descr_names_both_columns(): my_mask.create_descr() assert my_mask._descr == "!=-99 in mag_z or mag_z2" + + +def test_covered_2bands_pairs_each_bit_with_its_own_band(): + + # Footprint bits (set = no imaging) in one group, magnitudes in the other + bits = np.array( + [(False, True), (False, True), (True, False), (True, True), (False, False)], + dtype=[("256_z", "?"), ("2048_z2", "?")], + ) + mags = np.array( + [(21.0, 20.0), (-99, 20.0), (21.0, 20.0), (21.0, 20.0), (-99, -99)], + dtype=[("mag_z", "f8"), ("mag_z2", "f8")], + ) + + my_mask = Mask( + "256_z", + "zband", + kind="covered_2bands", + value=-99, + col_name2="2048_z2", + mag_col="mag_z", + mag_col2="mag_z2", + dat=bits, + dat_other=mags, + ) + + # row 0: z measured and covered -> keep + # row 1: z absent, z2 measured but not covered -> drop + # row 2: z not covered, z2 measured and covered -> keep + # row 3: neither band covered -> drop + # row 4: both covered but neither measured -> drop + npt.assert_array_equal(my_mask._mask, [True, False, True, False, False]) + + +def test_covered_2bands_requires_its_columns_and_other_group(): + + with pytest.raises(ValueError, match="mag_col"): + Mask("256_z", "zband", kind="covered_2bands", value=-99, col_name2="2048_z2") + + with pytest.raises(ValueError, match="dat_other"): + Mask( + "256_z", + "zband", + kind="covered_2bands", + value=-99, + col_name2="2048_z2", + mag_col="mag_z", + mag_col2="mag_z2", + ) From 1c88cf2fe7c03d99375978fb2e81e917d67b42ab Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sun, 20 Sep 2026 18:18:34 +0200 Subject: [PATCH 11/14] Resolve conflict markers left in the upstream/develop merge Merge 6ec03ae committed scripts/fill_photoz_bands.py and scripts/check_filled_fields.py with their conflict markers still in place, so neither file could be imported or run. Both were added independently on the two sides, which is why git conflicted the whole file. Upstream's version is a superset: it adds the per-tile row-order check (check_tile_row_order, n_check_rows, check_tol_arcsec, FITS_RADEC_CANDIDATES) and the Z_ML key, and differs otherwise only in formatting. Taken as-is, with the one local change upstream lacks re-applied: reading TILE_ID with dset.fields() so the scan chunk holds one column instead of all of them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013NzWDtdQbsK1VhvYbDnmTo --- scripts/check_filled_fields.py | 12 ++----- scripts/fill_photoz_bands.py | 64 ---------------------------------- 2 files changed, 2 insertions(+), 74 deletions(-) diff --git a/scripts/check_filled_fields.py b/scripts/check_filled_fields.py index 8b3e62f6..675ad116 100644 --- a/scripts/check_filled_fields.py +++ b/scripts/check_filled_fields.py @@ -10,8 +10,8 @@ HDF5_FILE = "unions_shapepipe_comprehensive_struc_ugriz_2024_v1.6.c.DR6.hdf5" EMPTY_VALUE = -199 -CHUNK_SIZE = 5_000_000 # rows per chunk -REPORT_EVERY = 10 # print running fractions every N chunks +CHUNK_SIZE = 5_000_000 # rows per chunk +REPORT_EVERY = 10 # print running fractions every N chunks FIELDS = [ "Z_B", @@ -54,24 +54,16 @@ # Periodic running-fraction report if (chunk_idx + 1) % REPORT_EVERY == 0 or (chunk_idx + 1) == n_chunks: rows_done = end -<<<<<<< HEAD - tqdm.write(f"\n --- after {rows_done:,} rows ({100*rows_done/n_total:.1f}%) ---") -======= tqdm.write( f"\n --- after {rows_done:,} rows ({100 * rows_done / n_total:.1f}%) ---" ) ->>>>>>> upstream/develop tqdm.write(f" {'Field':<22} {'Filled %':>9}") for field in FIELDS: pct = 100.0 * counts[field] / rows_done tqdm.write(f" {field:<22} {pct:>8.2f}%") # Final summary -<<<<<<< HEAD -print(f"\n{'='*56}") -======= print(f"\n{'=' * 56}") ->>>>>>> upstream/develop print(f"FINAL SUMMARY (total rows: {n_total:,})") print(f"{'Field':<22} {'Filled':>12} {'Empty':>12} {'Filled %':>10}") print("-" * 60) diff --git a/scripts/fill_photoz_bands.py b/scripts/fill_photoz_bands.py index d65c7d42..f0810b05 100755 --- a/scripts/fill_photoz_bands.py +++ b/scripts/fill_photoz_bands.py @@ -35,18 +35,6 @@ import numpy as np import tqdm from astropy.io import fits -<<<<<<< HEAD - -from cs_util import args as cs_args -from cs_util import logging - - -FITS_HDU = 1 -EMPTY_VALUE = -199 -COPY_CHUNK = 2_000_000 # rows per chunk when copying input → output -SCAN_CHUNK = 5_000_000 # rows per chunk when scanning TILE_ID -MAX_CONSEC_FAILS = 10 # abort if this many tiles in a row fail -======= from cs_util import args as cs_args from cs_util import logging @@ -70,17 +58,13 @@ ("RA", "Dec"), ("X_WORLD", "Y_WORLD"), ] ->>>>>>> upstream/develop REQUESTED_KEYS = [ "Z_B", "Z_B_MIN", "Z_B_MAX", "T_B", -<<<<<<< HEAD -======= "Z_ML", ->>>>>>> upstream/develop "MAG_GAAP_u", "MAGERR_GAAP_u", "MAG_GAAP_0p7_u", @@ -174,11 +158,8 @@ def params_default(): "output": "unions_shapepipe_comprehensive_struc_ugriz_2024_v1.5.c.hdf5", "fits_dir": "UNIONS_DR6", "checkpoint": "fill_photoz_bands_checkpoint.json", -<<<<<<< HEAD -======= "n_check_rows": 10, "check_tol_arcsec": 0.5, ->>>>>>> upstream/develop "verbose": False, } @@ -189,22 +170,16 @@ def params_default(): "checkpoint": "-c", } -<<<<<<< HEAD - types = {} -======= types = { "n_check_rows": "int", "check_tol_arcsec": "float", } ->>>>>>> upstream/develop help_strings = { "input": "input HDF5 catalogue (no PhotoPipe fields), default={}", "output": "output HDF5 catalogue (PhotoPipe fields added and filled), default={}", "fits_dir": "directory with PhotoPipe FITS tiles, default={}", "checkpoint": "checkpoint JSON file for resume support, default={}", -<<<<<<< HEAD -======= "n_check_rows": ( "number of rows per tile whose RA/Dec are compared between HDF5 and" " FITS to verify row order, 0 to disable, default={}" @@ -213,7 +188,6 @@ def params_default(): "maximum angular separation [arcsec] for a row-order check to pass," " default={}" ), ->>>>>>> upstream/develop } return params, short_options, types, help_strings @@ -223,10 +197,7 @@ def params_default(): # Helpers # --------------------------------------------------------------------------- -<<<<<<< HEAD -======= ->>>>>>> upstream/develop def detect_dataset_name(hf): """Return the first dataset name in an HDF5 file. @@ -319,8 +290,6 @@ def check_fits_keys(fits_columns, requested_keys): return valid, missing -<<<<<<< HEAD -======= def find_fits_radec_columns(fits_columns): """Find RA/Dec Columns. @@ -498,20 +467,16 @@ def check_tile_row_order( return bool(max_sep <= tol_arcsec), n_checked, max_sep ->>>>>>> upstream/develop def write_tile_to_hdf5(dset, hdf5_indices, fits_data, valid_keys): """Write valid_keys from fits_data into dset at hdf5_indices. Reads the HDF5 range in one chunk, fills fields in memory, writes back. Handles both contiguous and non-contiguous index ranges. -<<<<<<< HEAD -======= Assumes row ``k`` of ``fits_data`` corresponds to HDF5 row ``numpy.sort(hdf5_indices)[k]``; ``check_tile_row_order`` spot-checks this before the write. ->>>>>>> upstream/develop Parameters ---------- dset : h5py.Dataset @@ -554,10 +519,7 @@ def write_tile_to_hdf5(dset, hdf5_indices, fits_data, valid_keys): # Phase 1: create output file # --------------------------------------------------------------------------- -<<<<<<< HEAD -======= ->>>>>>> upstream/develop def create_output_file(input_path, output_path, dataset_name, verbose=False): """Create output HDF5 by copying input and appending empty PhotoPipe fields. @@ -623,10 +585,7 @@ def create_output_file(input_path, output_path, dataset_name, verbose=False): # Main # --------------------------------------------------------------------------- -<<<<<<< HEAD -======= ->>>>>>> upstream/develop def main(argv=None): """Main. @@ -722,13 +681,9 @@ def main(argv=None): ) as pbar: for start in range(0, n_total, SCAN_CHUNK): end = min(start + SCAN_CHUNK, n_total) -<<<<<<< HEAD # .fields() reads only TILE_ID; dset[start:end]["TILE_ID"] # would materialise all columns first (~3 GB per chunk) tile_chunk = dset.fields("TILE_ID")[start:end] -======= - tile_chunk = dset[start:end]["TILE_ID"] ->>>>>>> upstream/develop for local_i, tid in enumerate(tile_chunk): tile_index_map_lists[tid].append(start + local_i) pbar.update(end - start) @@ -742,12 +697,6 @@ def main(argv=None): unique_tiles = sorted(tile_index_map.keys()) n_tiles = len(unique_tiles) -<<<<<<< HEAD - valid_keys = None # determined from first available FITS tile - n_skipped_missing = 0 - n_skipped_done = 0 - n_skipped_size = 0 -======= valid_keys = None # determined from first available FITS tile fits_radec_cols = None # idem, (RA, Dec) column names in the FITS tiles @@ -763,7 +712,6 @@ def main(argv=None): n_skipped_done = 0 n_skipped_size = 0 n_skipped_order = 0 ->>>>>>> upstream/develop n_errors = 0 n_consec_fails = 0 n_processed = 0 @@ -812,8 +760,6 @@ def main(argv=None): ) tqdm.tqdm.write(f"\n Keys to fill: {valid_keys}\n") -<<<<<<< HEAD -======= fits_radec_cols = find_fits_radec_columns(fits_data.dtype.names) if check_rows > 0 and fits_radec_cols[0] is None: warnings.warn( @@ -832,7 +778,6 @@ def main(argv=None): f" {params['check_tol_arcsec']} arcsec\n" ) ->>>>>>> upstream/develop hdf5_indices = tile_index_map[tile_id] n_hdf5 = len(hdf5_indices) n_fits = len(fits_data) @@ -849,9 +794,6 @@ def main(argv=None): ) continue -<<<<<<< HEAD - write_tile_to_hdf5(dset, hdf5_indices, fits_data, valid_keys) -======= # write_tile_to_hdf5 pairs FITS row k with HDF5 row # sorted_idx[k]; spot-check that pairing before writing. sorted_idx = np.sort(hdf5_indices) @@ -890,7 +832,6 @@ def main(argv=None): continue write_tile_to_hdf5(dset, sorted_idx, fits_data, valid_keys) ->>>>>>> upstream/develop except Exception as e: warnings.warn(f"Tile {tile_str}: error ({e}), skipping.") @@ -920,14 +861,9 @@ def main(argv=None): print(f" Tiles skipped (done) : {n_skipped_done}") print(f" FITS files missing : {n_skipped_missing}") print(f" Size mismatches : {n_skipped_size}") -<<<<<<< HEAD - print(f" Tiles failed (error) : {n_errors}") - if n_processed == 0 and (n_errors > 0 or n_skipped_size > 0): -======= print(f" Row-order mismatches : {n_skipped_order}") print(f" Tiles failed (error) : {n_errors}") if n_processed == 0 and (n_errors > 0 or n_skipped_size > 0 or n_skipped_order > 0): ->>>>>>> upstream/develop print( "WARNING: no tiles were filled; all available tiles failed.", file=sys.stderr, From dfe6ee00740a04173c88af28ccbd269ea4976dc4 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Wed, 23 Sep 2026 11:53:43 +0200 Subject: [PATCH 12/14] added forgotten PhotoPipe columns for calibrated cat --- .../calibrate_comprehensive_cat.py | 12 +++ scripts/fill_photoz_bands.py | 85 ++----------------- src/sp_validation/calibration.py | 7 +- src/sp_validation/catalog.py | 29 +++++++ 4 files changed, 52 insertions(+), 81 deletions(-) diff --git a/scripts/calibration/calibrate_comprehensive_cat.py b/scripts/calibration/calibrate_comprehensive_cat.py index b9d44e2d..c46679af 100644 --- a/scripts/calibration/calibrate_comprehensive_cat.py +++ b/scripts/calibration/calibrate_comprehensive_cat.py @@ -192,6 +192,18 @@ "NGMIX_Tpsf_NOSHEAR", "fwhm_PSF", ] + +# PhotoPipe photo-z and multi-band photometry, when fill_photoz_bands.py has +# put them in the input. Selected against what the input actually holds, for +# two reasons: no catalogue carries every key (v1.6.c.3 has no Z_ML), and the +# image-simulation runs share this script with none of these columns at all. +photoz_cols = [key for key in cat.PHOTOZ_KEYS if key in set(dat.dtype.names or ())] +if photoz_cols: + print(f"Copying {len(photoz_cols)} PhotoPipe columns to the output catalogue") +else: + print("No PhotoPipe columns found in the input; none copied") +add_cols = add_cols + photoz_cols + add_cols_data = {} for key in add_cols: add_cols_data[key] = cat.get_col(dat, key, mask_combined._mask, mask_metacal) diff --git a/scripts/fill_photoz_bands.py b/scripts/fill_photoz_bands.py index f0810b05..70d64d0e 100755 --- a/scripts/fill_photoz_bands.py +++ b/scripts/fill_photoz_bands.py @@ -38,6 +38,8 @@ from cs_util import args as cs_args from cs_util import logging +import sp_validation.catalog as sp_cat + FITS_HDU = 1 EMPTY_VALUE = -199 COPY_CHUNK = 2_000_000 # rows per chunk when copying input → output @@ -59,86 +61,9 @@ ("X_WORLD", "Y_WORLD"), ] -REQUESTED_KEYS = [ - "Z_B", - "Z_B_MIN", - "Z_B_MAX", - "T_B", - "Z_ML", - "MAG_GAAP_u", - "MAGERR_GAAP_u", - "MAG_GAAP_0p7_u", - "MAGERR_GAAP_0p7_u", - "MAG_GAAP_1p0_u", - "MAGERR_GAAP_1p0_u", - "FLAG_GAAP_u", - "MAG_LIM_u", - "FLUX_GAAP_u", - "FLUXERR_GAAP_u", - "EXTINCTION_u", - "MAG_GAAP_g", - "MAGERR_GAAP_g", - "MAG_GAAP_0p7_g", - "MAGERR_GAAP_0p7_g", - "MAG_GAAP_1p0_g", - "MAGERR_GAAP_1p0_g", - "FLAG_GAAP_g", - "MAG_LIM_g", - "FLUX_GAAP_g", - "FLUXERR_GAAP_g", - "EXTINCTION_g", - "MAG_GAAP_r", - "MAGERR_GAAP_r", - "MAG_GAAP_0p7_r", - "MAGERR_GAAP_0p7_r", - "MAG_GAAP_1p0_r", - "MAGERR_GAAP_1p0_r", - "FLAG_GAAP_r", - "MAG_LIM_r", - "FLUX_GAAP_r", - "FLUXERR_GAAP_r", - "EXTINCTION_r", - "MAG_GAAP_i", - "MAGERR_GAAP_i", - "MAG_GAAP_0p7_i", - "MAGERR_GAAP_0p7_i", - "MAG_GAAP_1p0_i", - "MAGERR_GAAP_1p0_i", - "FLAG_GAAP_i", - "MAG_LIM_i", - "FLUX_GAAP_i", - "FLUXERR_GAAP_i", - "EXTINCTION_i", - "MAG_GAAP_z", - "MAGERR_GAAP_z", - "MAG_GAAP_0p7_z", - "MAGERR_GAAP_0p7_z", - "MAG_GAAP_1p0_z", - "MAGERR_GAAP_1p0_z", - "FLAG_GAAP_z", - "MAG_LIM_z", - "FLUX_GAAP_z", - "FLUXERR_GAAP_z", - "EXTINCTION_z", - "MAG_GAAP_z2", - "MAGERR_GAAP_z2", - "MAG_GAAP_0p7_z2", - "MAGERR_GAAP_0p7_z2", - "MAG_GAAP_1p0_z2", - "MAGERR_GAAP_1p0_z2", - "FLAG_GAAP_z2", - "MAG_LIM_z2", - "FLUX_GAAP_z2", - "FLUXERR_GAAP_z2", - "EXTINCTION_z2", - "EXTINCTION", - "ODDS", - "CHI_SQUARED_BPZ", - "M_0", - "BPZ_FILT", - "BPZ_NONDETFILT", - "BPZ_FLAGFILT", -] +# The one list, shared with the calibration script so the two cannot +# drift (see sp_validation.catalog.PHOTOZ_KEYS). +REQUESTED_KEYS = sp_cat.PHOTOZ_KEYS def params_default(): diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index 6147aa0e..72414709 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -184,8 +184,13 @@ def fill_cat_gal(cat_gal, dat, g_uncorr, gal_metacal, mask1, mask2, purpose="wei cat_gal["R_g22"] = gal_metacal.R22 cat_gal["NGMIX_T_NOSHEAR"] = sp_cat.get_col(dat, "NGMIX_T_NOSHEAR", mask1, mask2) + # v1.6 writes NGMIX_Tpsf_NOSHEAR, ShapePipe v2 NGMIX_T_PSF_RECONV_NOSHEAR; + # the output column keeps one name either way. cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] = sp_cat.get_col( - dat, "NGMIX_T_PSF_RECONV_NOSHEAR", mask1, mask2 + dat, + first_present(dat, "NGMIX_T_PSF_RECONV_NOSHEAR", "NGMIX_Tpsf_NOSHEAR"), + mask1, + mask2, ) cat_gal["size_ratio"] = ( cat_gal["NGMIX_T_NOSHEAR"] / cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] diff --git a/src/sp_validation/catalog.py b/src/sp_validation/catalog.py index e39d2076..452316c4 100644 --- a/src/sp_validation/catalog.py +++ b/src/sp_validation/catalog.py @@ -27,6 +27,35 @@ from sp_validation.version import __version__ +# PhotoPipe photo-z and multi-band photometry columns, as added to a +# comprehensive catalogue by scripts/fill_photoz_bands.py and copied on to the +# cut catalogue by scripts/calibration/calibrate_comprehensive_cat.py. ONE list +# for both: when it lived only in the filling script, the calibration script +# had no idea these columns existed and dropped all 77 of them from +# v1.6.6. Not every catalogue carries every key (v1.6.c.3 has no Z_ML), and +# image-sim catalogues carry none, so consumers select on what the input holds. +PHOTOZ_KEYS = [ + "Z_B", "Z_B_MIN", "Z_B_MAX", "T_B", "Z_ML", "MAG_GAAP_u", "MAGERR_GAAP_u", + "MAG_GAAP_0p7_u", "MAGERR_GAAP_0p7_u", "MAG_GAAP_1p0_u", "MAGERR_GAAP_1p0_u", + "FLAG_GAAP_u", "MAG_LIM_u", "FLUX_GAAP_u", "FLUXERR_GAAP_u", "EXTINCTION_u", + "MAG_GAAP_g", "MAGERR_GAAP_g", "MAG_GAAP_0p7_g", "MAGERR_GAAP_0p7_g", + "MAG_GAAP_1p0_g", "MAGERR_GAAP_1p0_g", "FLAG_GAAP_g", "MAG_LIM_g", + "FLUX_GAAP_g", "FLUXERR_GAAP_g", "EXTINCTION_g", "MAG_GAAP_r", "MAGERR_GAAP_r", + "MAG_GAAP_0p7_r", "MAGERR_GAAP_0p7_r", "MAG_GAAP_1p0_r", "MAGERR_GAAP_1p0_r", + "FLAG_GAAP_r", "MAG_LIM_r", "FLUX_GAAP_r", "FLUXERR_GAAP_r", "EXTINCTION_r", + "MAG_GAAP_i", "MAGERR_GAAP_i", "MAG_GAAP_0p7_i", "MAGERR_GAAP_0p7_i", + "MAG_GAAP_1p0_i", "MAGERR_GAAP_1p0_i", "FLAG_GAAP_i", "MAG_LIM_i", + "FLUX_GAAP_i", "FLUXERR_GAAP_i", "EXTINCTION_i", "MAG_GAAP_z", "MAGERR_GAAP_z", + "MAG_GAAP_0p7_z", "MAGERR_GAAP_0p7_z", "MAG_GAAP_1p0_z", "MAGERR_GAAP_1p0_z", + "FLAG_GAAP_z", "MAG_LIM_z", "FLUX_GAAP_z", "FLUXERR_GAAP_z", "EXTINCTION_z", + "MAG_GAAP_z2", "MAGERR_GAAP_z2", "MAG_GAAP_0p7_z2", "MAGERR_GAAP_0p7_z2", + "MAG_GAAP_1p0_z2", "MAGERR_GAAP_1p0_z2", "FLAG_GAAP_z2", "MAG_LIM_z2", + "FLUX_GAAP_z2", "FLUXERR_GAAP_z2", "EXTINCTION_z2", "EXTINCTION", + "ODDS", "CHI_SQUARED_BPZ", "M_0", "BPZ_FILT", "BPZ_NONDETFILT", + "BPZ_FLAGFILT", +] + + def print_mean_ellipticity( dd, ell_col_name, From eaeb5e9c8c07734719d2d579b20e5305942932e8 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Thu, 8 Oct 2026 13:29:54 +0200 Subject: [PATCH 13/14] fixed v1.X.6 pp mask --- config/calibration/mask_v1.X.6_ppv1.yaml | 11 +++++++++++ scripts/examples/create_binned_mask_comprehensive.py | 2 +- workflow/profiles/candide/config.yaml | 3 ++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/config/calibration/mask_v1.X.6_ppv1.yaml b/config/calibration/mask_v1.X.6_ppv1.yaml index 0a7dc471..9ccb9205 100644 --- a/config/calibration/mask_v1.X.6_ppv1.yaml +++ b/config/calibration/mask_v1.X.6_ppv1.yaml @@ -169,3 +169,14 @@ metacal: # Weight for global response matrix, None for unweighted mean global_R_weight: w + +# Binned footprint mask +binned_mask: + # Resolution + nside: 8192 + + # Mask value for "good" (observed) area + good: False + + # Output file path base + output_base: "footprint_binned" diff --git a/scripts/examples/create_binned_mask_comprehensive.py b/scripts/examples/create_binned_mask_comprehensive.py index 68964120..bb4846fa 100644 --- a/scripts/examples/create_binned_mask_comprehensive.py +++ b/scripts/examples/create_binned_mask_comprehensive.py @@ -32,7 +32,7 @@ # %% # Read configuration file and set parameters -config = obj.read_config_set_params("config_for_compr_mask.yaml") +config = obj.read_config_set_params("config_mask.yaml") # %% # Get data. Set load_into_memory to False for very large files diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 19afd89a..1f9afd02 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -60,7 +60,8 @@ default-resources: slurm_partition: "comp,pscomp" runtime: "60m" cpus_per_task: 12 - slurm_extra: "'--exclude=n17,n09,n36'" + # n35: grid_3 image-sims jobs filled its /tmp (2026-09-30); never use it. + slurm_extra: "'--exclude=n17,n09,n35,n36'" # Retry a job once on transient node failure, and keep the SLURM logs of # successful jobs (candide debugging). From b2208d41882dc3ab3d4d46cbc24e8dea1e78c8d9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:42:47 +0000 Subject: [PATCH 14/14] ruff autofix (format + safe lint fixes) Pushed by the lint gate. --- scripts/calibration/demo_apply_hsp_masks.py | 1 - src/sp_validation/catalog.py | 95 +++++++++++++++++---- 2 files changed, 77 insertions(+), 19 deletions(-) diff --git a/scripts/calibration/demo_apply_hsp_masks.py b/scripts/calibration/demo_apply_hsp_masks.py index 679080ed..f5056a06 100644 --- a/scripts/calibration/demo_apply_hsp_masks.py +++ b/scripts/calibration/demo_apply_hsp_masks.py @@ -38,7 +38,6 @@ obj = sp_joint.ApplyHspMasks() - # + # Set parameters base = "unions_shapepipe_comprehensive" diff --git a/src/sp_validation/catalog.py b/src/sp_validation/catalog.py index ca1596fe..ec073dcb 100644 --- a/src/sp_validation/catalog.py +++ b/src/sp_validation/catalog.py @@ -24,7 +24,6 @@ from sp_validation import format, io from sp_validation.version import __version__ - # PhotoPipe photo-z and multi-band photometry columns, as added to a # comprehensive catalogue by scripts/fill_photoz_bands.py and copied on to the # cut catalogue by scripts/calibration/calibrate_comprehensive_cat.py. ONE list @@ -33,23 +32,83 @@ # v1.6.6. Not every catalogue carries every key (v1.6.c.3 has no Z_ML), and # image-sim catalogues carry none, so consumers select on what the input holds. PHOTOZ_KEYS = [ - "Z_B", "Z_B_MIN", "Z_B_MAX", "T_B", "Z_ML", "MAG_GAAP_u", "MAGERR_GAAP_u", - "MAG_GAAP_0p7_u", "MAGERR_GAAP_0p7_u", "MAG_GAAP_1p0_u", "MAGERR_GAAP_1p0_u", - "FLAG_GAAP_u", "MAG_LIM_u", "FLUX_GAAP_u", "FLUXERR_GAAP_u", "EXTINCTION_u", - "MAG_GAAP_g", "MAGERR_GAAP_g", "MAG_GAAP_0p7_g", "MAGERR_GAAP_0p7_g", - "MAG_GAAP_1p0_g", "MAGERR_GAAP_1p0_g", "FLAG_GAAP_g", "MAG_LIM_g", - "FLUX_GAAP_g", "FLUXERR_GAAP_g", "EXTINCTION_g", "MAG_GAAP_r", "MAGERR_GAAP_r", - "MAG_GAAP_0p7_r", "MAGERR_GAAP_0p7_r", "MAG_GAAP_1p0_r", "MAGERR_GAAP_1p0_r", - "FLAG_GAAP_r", "MAG_LIM_r", "FLUX_GAAP_r", "FLUXERR_GAAP_r", "EXTINCTION_r", - "MAG_GAAP_i", "MAGERR_GAAP_i", "MAG_GAAP_0p7_i", "MAGERR_GAAP_0p7_i", - "MAG_GAAP_1p0_i", "MAGERR_GAAP_1p0_i", "FLAG_GAAP_i", "MAG_LIM_i", - "FLUX_GAAP_i", "FLUXERR_GAAP_i", "EXTINCTION_i", "MAG_GAAP_z", "MAGERR_GAAP_z", - "MAG_GAAP_0p7_z", "MAGERR_GAAP_0p7_z", "MAG_GAAP_1p0_z", "MAGERR_GAAP_1p0_z", - "FLAG_GAAP_z", "MAG_LIM_z", "FLUX_GAAP_z", "FLUXERR_GAAP_z", "EXTINCTION_z", - "MAG_GAAP_z2", "MAGERR_GAAP_z2", "MAG_GAAP_0p7_z2", "MAGERR_GAAP_0p7_z2", - "MAG_GAAP_1p0_z2", "MAGERR_GAAP_1p0_z2", "FLAG_GAAP_z2", "MAG_LIM_z2", - "FLUX_GAAP_z2", "FLUXERR_GAAP_z2", "EXTINCTION_z2", "EXTINCTION", - "ODDS", "CHI_SQUARED_BPZ", "M_0", "BPZ_FILT", "BPZ_NONDETFILT", + "Z_B", + "Z_B_MIN", + "Z_B_MAX", + "T_B", + "Z_ML", + "MAG_GAAP_u", + "MAGERR_GAAP_u", + "MAG_GAAP_0p7_u", + "MAGERR_GAAP_0p7_u", + "MAG_GAAP_1p0_u", + "MAGERR_GAAP_1p0_u", + "FLAG_GAAP_u", + "MAG_LIM_u", + "FLUX_GAAP_u", + "FLUXERR_GAAP_u", + "EXTINCTION_u", + "MAG_GAAP_g", + "MAGERR_GAAP_g", + "MAG_GAAP_0p7_g", + "MAGERR_GAAP_0p7_g", + "MAG_GAAP_1p0_g", + "MAGERR_GAAP_1p0_g", + "FLAG_GAAP_g", + "MAG_LIM_g", + "FLUX_GAAP_g", + "FLUXERR_GAAP_g", + "EXTINCTION_g", + "MAG_GAAP_r", + "MAGERR_GAAP_r", + "MAG_GAAP_0p7_r", + "MAGERR_GAAP_0p7_r", + "MAG_GAAP_1p0_r", + "MAGERR_GAAP_1p0_r", + "FLAG_GAAP_r", + "MAG_LIM_r", + "FLUX_GAAP_r", + "FLUXERR_GAAP_r", + "EXTINCTION_r", + "MAG_GAAP_i", + "MAGERR_GAAP_i", + "MAG_GAAP_0p7_i", + "MAGERR_GAAP_0p7_i", + "MAG_GAAP_1p0_i", + "MAGERR_GAAP_1p0_i", + "FLAG_GAAP_i", + "MAG_LIM_i", + "FLUX_GAAP_i", + "FLUXERR_GAAP_i", + "EXTINCTION_i", + "MAG_GAAP_z", + "MAGERR_GAAP_z", + "MAG_GAAP_0p7_z", + "MAGERR_GAAP_0p7_z", + "MAG_GAAP_1p0_z", + "MAGERR_GAAP_1p0_z", + "FLAG_GAAP_z", + "MAG_LIM_z", + "FLUX_GAAP_z", + "FLUXERR_GAAP_z", + "EXTINCTION_z", + "MAG_GAAP_z2", + "MAGERR_GAAP_z2", + "MAG_GAAP_0p7_z2", + "MAGERR_GAAP_0p7_z2", + "MAG_GAAP_1p0_z2", + "MAGERR_GAAP_1p0_z2", + "FLAG_GAAP_z2", + "MAG_LIM_z2", + "FLUX_GAAP_z2", + "FLUXERR_GAAP_z2", + "EXTINCTION_z2", + "EXTINCTION", + "ODDS", + "CHI_SQUARED_BPZ", + "M_0", + "BPZ_FILT", + "BPZ_NONDETFILT", "BPZ_FLAGFILT", ]