diff --git a/src/sp_validation/catalog.py b/src/sp_validation/catalog.py index b590993f..8ea595d7 100644 --- a/src/sp_validation/catalog.py +++ b/src/sp_validation/catalog.py @@ -235,8 +235,9 @@ def check_invalid(dd, key, val, stats_file, name=None, verbose=False): for i in range(len(key)): w = dd[key[i]] == val[i] n_inv_psf = len(np.where(w)[0]) - msg = "Invalid {} found for {}/{} = {:.1g}% objects".format( - name[i], n_inv_psf, n_all, n_inv_psf / n_all + msg = ( + f"Invalid {name[i]} found for {n_inv_psf}/{n_all}" + + f" = {n_inv_psf / n_all:.3%} objects" ) io.print_stats(msg, stats_file, verbose=verbose) @@ -523,6 +524,10 @@ def write_shape_catalog( fields = [] for col, _ in col_info_arr: arr = np.asarray(col.array) + if str(col.format) == "L": + # astropy holds FITS logicals as the characters 'T'/'F' + # (84/70); write booleans, not those codes. + arr = arr == ord("T") if arr.ndim == 2: for idx in range(arr.shape[1]): fields.append((f"{col.name}_{idx}", arr[:, idx])) diff --git a/src/sp_validation/catalog_builders.py b/src/sp_validation/catalog_builders.py index 630cdc61..04abc4f6 100644 --- a/src/sp_validation/catalog_builders.py +++ b/src/sp_validation/catalog_builders.py @@ -159,7 +159,7 @@ def write_hdf5_header(self, hd5file): input HDF5 file """ - author = os.getenv("USER") + author = os.getenv("USER", "unknown") software_name = "sp_validation" software_version = version(software_name) date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/src/sp_validation/tests/test_catalog_flag_roundtrip.py b/src/sp_validation/tests/test_catalog_flag_roundtrip.py index 8272d934..39e37778 100644 --- a/src/sp_validation/tests/test_catalog_flag_roundtrip.py +++ b/src/sp_validation/tests/test_catalog_flag_roundtrip.py @@ -80,3 +80,44 @@ def test_reduce_mem_never_narrows_integers(reduce_mem): assert builder.dtype_out(name, np.dtype(">f8")) == np.dtype(">f8") expected = np.float32 if reduce_mem else np.dtype(">f8") assert builder.dtype_out("NGMIX_G1_NOSHEAR", np.dtype(">f8")) == expected + + +@pytest.mark.parametrize("extension", [".fits", ".hdf5"]) +def test_mask_columns_roundtrip_as_booleans(tmp_path, extension): + """Logical (format "L") mask columns come back as booleans, True = masked. + + astropy holds FITS logicals as the characters 'T'/'F'; the HDF5 writer + must not store those character codes (84/70), which every mask cut + (``kind: equal, value: False``) would read as masked. + """ + with np.printoptions(): + params = runpy.run_path(str(ROOT / "scripts/calibration/params.py")) + masked = np.array([0, 1, 1, 0], dtype=np.int8) + columns = {name: masked for name in params["mask_columns"]} + assert all(params["add_cols_pre_cal_format"][name] == "L" for name in columns) + # Columns of other formats alongside, as in the real comprehensive catalogue + others = { + "TILE_ID": np.array([b"200.300", b"200.300", b"201.300", b"201.300"]), + "NUMBER": np.arange(len(masked)), + "FLAGS": np.array([0, 2, 3, 0]), + } + path = tmp_path / f"comprehensive{extension}" + write_shape_catalog( + str(path), + np.zeros(len(masked)), + np.zeros(len(masked)), + np.ones(len(masked)), + add_cols=columns | others, + add_cols_format=params["add_cols_pre_cal_format"], + ) + if extension == ".fits": + written = fits.getdata(path, 1) + else: + with h5py.File(path, "r") as catalog: + written = catalog["data"][:] + + np.testing.assert_array_equal(written["NUMBER"], others["NUMBER"]) + np.testing.assert_array_equal(written["FLAGS"], others["FLAGS"]) + for name in columns: + assert written[name].dtype == np.bool_, name + np.testing.assert_array_equal(written[name], masked.astype(bool), err_msg=name)